[PATCH v4] ash: add process substitution in bash-compatibility mode

Ron Yorston rmy at pobox.com
Tue Jan 29 08:49:52 UTC 2019


Process substitution is a Korn shell feature that's also available
in bash and some other shells.  This patch implements process
substitution in ash when ASH_BASH_COMPAT is enabled.

function                                             old     new   delta
argstr                                              1313    1477    +164
readtoken1                                          3265    3366    +101
close_new_fds                                          -      71     +71
strtodest                                              -      56     +56
evaltree                                             606     647     +41
.rodata                                           173923  173951     +28
cmdputs                                              393     418     +25
evalvar                                              697     714     +17
evalcommand                                         1724    1740     +16
cmdloop                                              404     411      +7
static.spclchars                                       9      11      +2
varvalue                                             669     626     -43
------------------------------------------------------------------------------
(add/remove: 2/0 grow/shrink: 9/1 up/down: 528/-43)           Total: 485 bytes
   text    data     bss     dec     hex filename
 928951    4199    1888  935038   e447e busybox_old
 929434    4199    1888  935521   e4661 busybox_unstripped

v2: Replace array of file descriptors with a linked list.
    Include tests that were unaccountably omitted from v1.
v3: Update linked list code to the intended version.
v4: Change order of conditional code in cmdputs().

Signed-off-by: Ron Yorston <rmy at pobox.com>
---
 include/platform.h                           |   2 +
 shell/ash.c                                  | 255 ++++++++++++++++---
 shell/ash_test/ash-psubst/bash_procsub.right |   9 +
 shell/ash_test/ash-psubst/bash_procsub.tests |  33 +++
 4 files changed, 257 insertions(+), 42 deletions(-)
 create mode 100644 shell/ash_test/ash-psubst/bash_procsub.right
 create mode 100755 shell/ash_test/ash-psubst/bash_procsub.tests

diff --git a/include/platform.h b/include/platform.h
index 50365a31c..7dd9551e2 100644
--- a/include/platform.h
+++ b/include/platform.h
@@ -417,6 +417,8 @@ typedef unsigned smalluint;
 #define HAVE_NET_ETHERNET_H 1
 #define HAVE_SYS_STATFS_H 1
 #define HAVE_PRINTF_PERCENTM 1
+#define HAVE_DEV_FD 1
+#define DEV_FD_PREFIX "/dev/fd/"
 
 #if defined(__UCLIBC__)
 # if UCLIBC_VERSION < KERNEL_VERSION(0, 9, 32)
diff --git a/shell/ash.c b/shell/ash.c
index a284b084d..82712aa13 100644
--- a/shell/ash.c
+++ b/shell/ash.c
@@ -224,6 +224,14 @@
 #define    BASH_XTRACEFD        ENABLE_ASH_BASH_COMPAT
 #define    BASH_READ_D          ENABLE_ASH_BASH_COMPAT
 #define IF_BASH_READ_D              IF_ASH_BASH_COMPAT
+/* <(...) and >(...) */
+#if HAVE_DEV_FD
+# define    BASH_PROCESS_SUBST   ENABLE_ASH_BASH_COMPAT
+# define IF_BASH_PROCESS_SUBST       IF_ASH_BASH_COMPAT
+#else
+# define    BASH_PROCESS_SUBST 0
+# define IF_BASH_PROCESS_SUBST(...)
+#endif
 
 #if defined(__ANDROID_API__) && __ANDROID_API__ <= 24
 /* Bionic at least up to version 24 has no glob() */
@@ -356,6 +364,11 @@ struct jmploc {
 	jmp_buf loc;
 };
 
+struct psfd {
+	int fd;
+	struct psfd *next;
+};
+
 struct globals_misc {
 	uint8_t exitstatus;     /* exit status of last command */
 	uint8_t back_exitstatus;/* exit status of backquoted command */
@@ -365,7 +378,9 @@ struct globals_misc {
 	int shlvl;
 #define rootshell (!shlvl)
 	int errlinno;
-
+#if BASH_PROCESS_SUBST
+	struct psfd *psfdlist;
+#endif
 	char *minusc;  /* argument to -c option */
 
 	char *curdir; // = nullstr;     /* current working directory */
@@ -444,6 +459,7 @@ extern struct globals_misc *BB_GLOBAL_CONST ash_ptr_to_globals_misc;
 #define rootpid     (G_misc.rootpid    )
 #define shlvl       (G_misc.shlvl      )
 #define errlinno    (G_misc.errlinno   )
+#define psfdlist    (G_misc.psfdlist   )
 #define minusc      (G_misc.minusc     )
 #define curdir      (G_misc.curdir     )
 #define physdir     (G_misc.physdir    )
@@ -714,6 +730,12 @@ out2str(const char *p)
 #define CTLENDARI    ((unsigned char)'\207')
 #define CTLQUOTEMARK ((unsigned char)'\210')
 #define CTL_LAST CTLQUOTEMARK
+#if BASH_PROCESS_SUBST
+# define CTLTOPROC    ((unsigned char)'\211')
+# define CTLFROMPROC  ((unsigned char)'\212')
+# undef CTL_LAST
+# define CTL_LAST CTLFROMPROC
+#endif
 
 /* variable substitution byte (follows CTLVAR) */
 #define VSTYPE  0x0f            /* type of variable substitution */
@@ -985,6 +1007,10 @@ trace_puts_quoted(char *s)
 		case CTLESC: c = 'e'; goto backslash;
 		case CTLVAR: c = 'v'; goto backslash;
 		case CTLBACKQ: c = 'q'; goto backslash;
+#if BASH_PROCESS_SUBST
+		case CTLTOPROC: c = 'p'; goto backslash;
+		case CTLFROMPROC: c = 'P'; goto backslash;
+#endif
  backslash:
 			putc('\\', tracefile);
 			putc(c, tracefile);
@@ -1146,8 +1172,17 @@ sharg(union node *arg, FILE *fp)
 		case CTLENDVAR:
 			putc('}', fp);
 			break;
+#if BASH_PROCESS_SUBST
+		case CTLTOPROC:
+			putc('>', fp);
+			goto backq;
+		case CTLFROMPROC:
+			putc('<', fp);
+			goto backq;
+#endif
 		case CTLBACKQ:
 			putc('$', fp);
+ IF_BASH_PROCESS_SUBST(backq:)
 			putc('(', fp);
 			shtree(bqlist->n, -1, NULL, fp);
 			putc(')', fp);
@@ -3153,8 +3188,13 @@ static const uint8_t syntax_index_table[] ALIGN1 = {
 	/* 134 CTLARI       */ CCTL_CCTL_CCTL_CCTL,
 	/* 135 CTLENDARI    */ CCTL_CCTL_CCTL_CCTL,
 	/* 136 CTLQUOTEMARK */ CCTL_CCTL_CCTL_CCTL,
+#if BASH_PROCESS_SUBST
+	/* 137 CTLTOPROC    */ CCTL_CCTL_CCTL_CCTL,
+	/* 138 CTLFROMPROC  */ CCTL_CCTL_CCTL_CCTL,
+#else
 	/* 137      */ CWORD_CWORD_CWORD_CWORD,
 	/* 138      */ CWORD_CWORD_CWORD_CWORD,
+#endif
 	/* 139      */ CWORD_CWORD_CWORD_CWORD,
 	/* 140      */ CWORD_CWORD_CWORD_CWORD,
 	/* 141      */ CWORD_CWORD_CWORD_CWORD,
@@ -4723,9 +4763,24 @@ cmdputs(const char *s)
 			quoted >>= 1;
 			subtype = 0;
 			goto dostr;
+#if BASH_PROCESS_SUBST
+		case CTLBACKQ:
+			c = '$';
+			str = "(...)";
+			break;
+		case CTLTOPROC:
+			c = '>';
+			str = "(...)";
+			break;
+		case CTLFROMPROC:
+			c = '<';
+			str = "(...)";
+			break;
+#else
 		case CTLBACKQ:
 			str = "$(...)";
 			goto dostr;
+#endif
 #if ENABLE_FEATURE_SH_MATH
 		case CTLARI:
 			str = "$((";
@@ -6370,6 +6425,39 @@ exptilde(char *startp, char *p, int flags)
 	return startp;
 }
 
+#if BASH_PROCESS_SUBST
+static void
+add_fd_to_list(int fd)
+{
+	struct psfd *p = (struct psfd *)xmalloc(sizeof(struct psfd));
+	p->fd = fd;
+	p->next = psfdlist;
+	psfdlist = p;
+}
+
+static void
+close_new_fds(struct psfd *stop)
+{
+	struct psfd *p;
+
+	INT_OFF;
+	p = psfdlist;
+	while (p && p != stop) {
+		close(p->fd);
+		p = p->next;
+		free(psfdlist);
+		psfdlist = p;
+	}
+	INT_ON;
+}
+
+static void
+close_all_fds(void)
+{
+	close_new_fds(NULL);
+}
+#endif
+
 /*
  * Execute a command inside back quotes.  If it's a builtin command, we
  * want to save its output in a block obtained from malloc.  Otherwise
@@ -6403,10 +6491,20 @@ evaltreenr(union node *n, int flags)
 }
 
 static void FAST_FUNC
-evalbackcmd(union node *n, struct backcmd *result)
+evalbackcmd(union node *n, struct backcmd *result
+				IF_BASH_PROCESS_SUBST(, int ctl))
 {
 	int pip[2];
 	struct job *jp;
+#if BASH_PROCESS_SUBST
+	/* determine end of pipe used by parent (ip) and child (ic) */
+	const int ip = (ctl == CTLTOPROC);
+	const int ic = !(ctl == CTLTOPROC);
+#else
+	const int ctl = CTLBACKQ;
+	const int ip = 0;
+	const int ic = 1;
+#endif
 
 	result->fd = -1;
 	result->buf = NULL;
@@ -6418,15 +6516,17 @@ evalbackcmd(union node *n, struct backcmd *result)
 
 	if (pipe(pip) < 0)
 		ash_msg_and_raise_perror("can't create pipe");
-	jp = makejob(/*n,*/ 1);
-	if (forkshell(jp, n, FORK_NOJOB) == 0) {
+	/* process substitution uses NULL job/node, like openhere() */
+	jp = (ctl == CTLBACKQ) ? makejob(/*n,*/ 1) : NULL;
+	if (forkshell(jp, (ctl == CTLBACKQ) ? n : NULL, FORK_NOJOB) == 0) {
 		/* child */
 		FORCE_INT_ON;
-		close(pip[0]);
-		if (pip[1] != 1) {
-			/*close(1);*/
-			dup2_or_raise(pip[1], 1);
-			close(pip[1]);
+		close(pip[ip]);
+		/* ic is index of child end of pipe *and* fd to connect it to */
+		if (pip[ic] != ic) {
+			/*close(ic);*/
+			dup2_or_raise(pip[ic], ic);
+			close(pip[ic]);
 		}
 /* TODO: eflag clearing makes the following not abort:
  *  ash -c 'set -e; z=$(false;echo foo); echo $z'
@@ -6442,8 +6542,18 @@ evalbackcmd(union node *n, struct backcmd *result)
 		/* NOTREACHED */
 	}
 	/* parent */
-	close(pip[1]);
-	result->fd = pip[0];
+#if BASH_PROCESS_SUBST
+	if (ctl != CTLBACKQ) {
+		int fd = fcntl(pip[ip], F_DUPFD, 64);
+		if (fd > 0) {
+			close(pip[ip]);
+			pip[ip] = fd;
+		}
+		add_fd_to_list(pip[ip]);
+	}
+#endif
+	close(pip[ic]);
+	result->fd = pip[ip];
 	result->jp = jp;
 
  out:
@@ -6455,8 +6565,11 @@ evalbackcmd(union node *n, struct backcmd *result)
  * Expand stuff in backwards quotes.
  */
 static void
-expbackq(union node *cmd, int flag)
+expbackq(union node *cmd, int flag IF_BASH_PROCESS_SUBST(, int ctl))
 {
+#if !BASH_PROCESS_SUBST
+	const int ctl = CTLBACKQ;
+#endif
 	struct backcmd in;
 	int i;
 	char buf[128];
@@ -6469,29 +6582,34 @@ expbackq(union node *cmd, int flag)
 	INT_OFF;
 	startloc = expdest - (char *)stackblock();
 	pushstackmark(&smark, startloc);
-	evalbackcmd(cmd, &in);
+	evalbackcmd(cmd, &in IF_BASH_PROCESS_SUBST(, ctl));
 	popstackmark(&smark);
 
-	p = in.buf;
-	i = in.nleft;
-	if (i == 0)
-		goto read;
-	for (;;) {
-		memtodest(p, i, syntax, flag & QUOTES_ESC);
+	if (ctl == CTLBACKQ) {
+		p = in.buf;
+		i = in.nleft;
+		if (i == 0)
+			goto read;
+		for (;;) {
+			memtodest(p, i, syntax, flag & QUOTES_ESC);
  read:
-		if (in.fd < 0)
-			break;
-		i = nonblock_immune_read(in.fd, buf, sizeof(buf));
-		TRACE(("expbackq: read returns %d\n", i));
-		if (i <= 0)
-			break;
-		p = buf;
-	}
+			if (in.fd < 0)
+				break;
+			i = nonblock_immune_read(in.fd, buf, sizeof(buf));
+			TRACE(("expbackq: read returns %d\n", i));
+			if (i <= 0)
+				break;
+			p = buf;
+		}
 
-	free(in.buf);
-	if (in.fd >= 0) {
-		close(in.fd);
-		back_exitstatus = waitforjob(in.jp);
+		free(in.buf);
+		if (in.fd >= 0) {
+			close(in.fd);
+			back_exitstatus = waitforjob(in.jp);
+		}
+	} else {
+		sprintf(buf, DEV_FD_PREFIX"%d", in.fd);
+		strtodest(buf, BASESYNTAX, 0);
 	}
 	INT_ON;
 
@@ -6587,6 +6705,10 @@ argstr(char *p, int flags)
 		CTLESC,
 		CTLVAR,
 		CTLBACKQ,
+#if BASH_PROCESS_SUBST
+		CTLTOPROC,
+		CTLFROMPROC,
+#endif
 #if ENABLE_FEATURE_SH_MATH
 		CTLENDARI,
 #endif
@@ -6689,8 +6811,12 @@ argstr(char *p, int flags)
 			p = evalvar(p, flags | inquotes);
 			TRACE(("argstr: evalvar:'%s'\n", (char *)stackblock()));
 			goto start;
+#if BASH_PROCESS_SUBST
+		case CTLTOPROC:
+		case CTLFROMPROC:
+#endif
 		case CTLBACKQ:
-			expbackq(argbackq->n, flags | inquotes);
+			expbackq(argbackq->n, flags | inquotes IF_BASH_PROCESS_SUBST(, c));
 			argbackq = argbackq->next;
 			goto start;
 #if ENABLE_FEATURE_SH_MATH
@@ -7433,7 +7559,8 @@ evalvar(char *p, int flag)
 			unsigned char c = *p++;
 			if (c == CTLESC)
 				p++;
-			else if (c == CTLBACKQ) {
+			else if (c == CTLBACKQ
+				IF_BASH_PROCESS_SUBST(|| c == CTLTOPROC || c == CTLFROMPROC)) {
 				if (varlen >= 0)
 					argbackq = argbackq->next;
 			} else if (c == CTLVAR) {
@@ -9035,6 +9162,9 @@ evaltree(union node *n, int flags)
 	int checkexit = 0;
 	int (*evalfn)(union node *, int);
 	int status = 0;
+#if BASH_PROCESS_SUBST
+	struct psfd *procsub_stop = psfdlist;
+#endif
 
 	if (n == NULL) {
 		TRACE(("evaltree(NULL) called\n"));
@@ -9139,6 +9269,15 @@ evaltree(union node *n, int flags)
 		break;
 	}
  out:
+#if BASH_PROCESS_SUBST
+	/* If we're in a shell function we close new fds here.  Otherwise
+	 *   f() { while true; do cat <(echo hi); done }
+	 *   f
+	 * runs out of file descriptors.
+	 */
+	if (funcline != 0)
+		close_new_fds(procsub_stop);
+#endif
 	/* Order of checks below is important:
 	 * signal handlers trigger before exit caused by "set -e".
 	 */
@@ -10268,6 +10407,15 @@ evalcommand(union node *cmd, int flags)
 	unwindredir(redir_stop);
 	unwindfiles(file_stop);
 	unwindlocalvars(localvar_stop);
+#if BASH_PROCESS_SUBST
+	/* Only close fds if we aren't in a shell function.  If we don't
+	 * close fds here
+	 *    while true; do cat <(echo hi); done
+	 * runs out of file descriptors.
+	 */
+	if (funcline == 0)
+		close_all_fds();
+#endif
 	if (lastarg) {
 		/* dsl: I think this is intended to be used to support
 		 * '_' in 'vi' command mode during line editing...
@@ -12005,11 +12153,13 @@ realeofmark(const char *eofmark)
  * using goto's to implement the subroutine linkage.  The following macros
  * will run code that appears at the end of readtoken1.
  */
+enum { OLD, NEW, PSUB };
 #define CHECKEND()      {goto checkend; checkend_return:;}
 #define PARSEREDIR()    {goto parseredir; parseredir_return:;}
 #define PARSESUB()      {goto parsesub; parsesub_return:;}
-#define PARSEBACKQOLD() {oldstyle = 1; goto parsebackq; parsebackq_oldreturn:;}
-#define PARSEBACKQNEW() {oldstyle = 0; goto parsebackq; parsebackq_newreturn:;}
+#define PARSEBACKQOLD() {style = OLD; goto parsebackq; parsebackq_oldreturn:;}
+#define PARSEBACKQNEW() {style = NEW; goto parsebackq; parsebackq_newreturn:;}
+#define PARSEPROCSUB()  {style = PSUB; goto parsebackq; parsebackq_psreturn:;}
 #define PARSEARITH()    {goto parsearith; parsearith_return:;}
 static int
 readtoken1(int c, int syntax, char *eofmark, int striptabs)
@@ -12020,7 +12170,7 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
 	size_t len;
 	struct nodelist *bqlist;
 	smallint quotef;
-	smallint oldstyle;
+	smallint style;
 	smallint pssyntax;   /* we are expanding a prompt string */
 	IF_BASH_DOLLAR_SQUOTE(smallint bash_dollar_squote = 0;)
 	/* syntax stack */
@@ -12203,6 +12353,15 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
 						c = 0x100 + '>'; /* flag &> */
 					pungetc();
 				}
+#endif
+#if BASH_PROCESS_SUBST
+				if (c == '<' || c == '>') {
+					if (pgetc() == '(') {
+						PARSEPROCSUB();
+						break;
+					}
+					pungetc();
+				}
 #endif
 				goto endword;   /* exit outer loop */
 			}
@@ -12591,7 +12750,7 @@ parsebackq: {
 		memcpy(str, stackblock(), savelen);
 	}
 
-	if (oldstyle) {
+	if (style == OLD) {
 		/* We must read until the closing backquote, giving special
 		 * treatment to some slashes, and then push the string and
 		 * reread it as input, interpreting it normally.
@@ -12649,20 +12808,20 @@ parsebackq: {
 	*nlpp = stzalloc(sizeof(**nlpp));
 	/* (*nlpp)->next = NULL; - stzalloc did it */
 
-	if (oldstyle) {
+	if (style == OLD) {
 		saveprompt = doprompt;
 		doprompt = 0;
 	}
 
 	n = list(2);
 
-	if (oldstyle)
+	if (style == OLD)
 		doprompt = saveprompt;
 	else if (readtoken() != TRP)
 		raise_error_unexpected_syntax(TRP);
 
 	(*nlpp)->n = n;
-	if (oldstyle) {
+	if (style == OLD) {
 		/*
 		 * Start reading from old file again, ignoring any pushed back
 		 * tokens left from the backquote parsing
@@ -12677,9 +12836,18 @@ parsebackq: {
 		memcpy(out, str, savelen);
 		STADJUST(savelen, out);
 	}
-	USTPUTC(CTLBACKQ, out);
-	if (oldstyle)
+#if BASH_PROCESS_SUBST
+	if (style == PSUB)
+		USTPUTC(c == '<' ? CTLFROMPROC : CTLTOPROC, out);
+	else
+#endif
+		USTPUTC(CTLBACKQ, out);
+	if (style == OLD)
 		goto parsebackq_oldreturn;
+#if BASH_PROCESS_SUBST
+	else if (style == PSUB)
+		goto parsebackq_psreturn;
+#endif
 	goto parsebackq_newreturn;
 }
 
@@ -13115,6 +13283,9 @@ cmdloop(int top)
 #if JOBS
 		if (doing_jobctl)
 			showjobs(SHOW_CHANGED|SHOW_STDERR);
+#endif
+#if BASH_PROCESS_SUBST
+		close_all_fds();
 #endif
 		inter = 0;
 		if (iflag && top) {
diff --git a/shell/ash_test/ash-psubst/bash_procsub.right b/shell/ash_test/ash-psubst/bash_procsub.right
new file mode 100644
index 000000000..aa16a96be
--- /dev/null
+++ b/shell/ash_test/ash-psubst/bash_procsub.right
@@ -0,0 +1,9 @@
+hello 1
+hello 2
+hello 3
+<(echo "hello 0")
+hello 4
+HI THERE
+hello error
+hello error
+hello stderr
diff --git a/shell/ash_test/ash-psubst/bash_procsub.tests b/shell/ash_test/ash-psubst/bash_procsub.tests
new file mode 100755
index 000000000..63b836782
--- /dev/null
+++ b/shell/ash_test/ash-psubst/bash_procsub.tests
@@ -0,0 +1,33 @@
+# simplest case
+cat <(echo "hello 1")
+
+# can have more than one
+cat <(echo "hello 2") <(echo "hello 3")
+
+# doesn't work in quotes
+echo "<(echo \"hello 0\")"
+
+# process substitution can be nested inside command substitution
+echo $(cat <(echo "hello 4"))
+
+# example from http://wiki.bash-hackers.org/syntax/expansion/proc_subst
+# process substitutions can be passed to a function as parameters or
+# variables
+f() {
+	cat "$1" >"$x"
+}
+x=>(tr '[:lower:]' '[:upper:]') f <(echo 'hi there')
+
+# process substitution can be combined with redirection on exec
+rm -f err
+# save stderr
+exec 4>&2
+# copy stderr to a file
+exec 2> >(tee err)
+echo "hello error" >&2
+sync
+# restore stderr
+exec 2>&4
+cat err
+rm -f err
+echo "hello stderr" >&2
-- 
2.20.1



More information about the busybox mailing list