[PATCH v5] ash: add process substitution in bash-compatibility mode

Ron Yorston rmy at pobox.com
Wed Jan 30 16:14:45 UTC 2019


Process substitution is a Korn shell feature that's also available
in bash and some other shells.  This patch implements process
substitution in ash when ASH_BASH_COMPAT is enabled.

function                                             old     new   delta
argstr                                              1313    1491    +178
readtoken1                                          3265    3366    +101
strtodest                                              -      56     +56
unwindredir                                            -      28     +28
.rodata                                           173923  173951     +28
cmdputs                                              393     418     +25
evalvar                                              697     714     +17
cmdloop                                              404     411      +7
static.spclchars                                       9      11      +2
pushredir                                            112      99     -13
evalcommand                                         1724    1711     -13
ash_main                                            1346    1332     -14
varvalue                                             669     626     -43
------------------------------------------------------------------------------
(add/remove: 2/0 grow/shrink: 7/4 up/down: 442/-83)           Total: 359 bytes
   text	   data	    bss	    dec	    hex	filename
 928914	   4199	   1888	 935001	  e4459	busybox_old
 929271	   4199	   1888	 935358	  e45be	busybox_unstripped

v2: Replace array of file descriptors with a linked list.
    Include tests that were unaccountably omitted from v1.
v3: Update linked list code to the intended version.
v4: Change order of conditional code in cmdputs().
v5: Use existing popredir() mechanism to manage file descriptors.

Signed-off-by: Ron Yorston <rmy at pobox.com>
---
 include/platform.h                           |   2 +
 shell/ash.c                                  | 214 +++++++++++++++----
 shell/ash_test/ash-psubst/bash_procsub.right |   9 +
 shell/ash_test/ash-psubst/bash_procsub.tests |  33 +++
 4 files changed, 213 insertions(+), 45 deletions(-)
 create mode 100644 shell/ash_test/ash-psubst/bash_procsub.right
 create mode 100755 shell/ash_test/ash-psubst/bash_procsub.tests

diff --git a/include/platform.h b/include/platform.h
index 50365a31c..7dd9551e2 100644
--- a/include/platform.h
+++ b/include/platform.h
@@ -417,6 +417,8 @@ typedef unsigned smalluint;
 #define HAVE_NET_ETHERNET_H 1
 #define HAVE_SYS_STATFS_H 1
 #define HAVE_PRINTF_PERCENTM 1
+#define HAVE_DEV_FD 1
+#define DEV_FD_PREFIX "/dev/fd/"
 
 #if defined(__UCLIBC__)
 # if UCLIBC_VERSION < KERNEL_VERSION(0, 9, 32)
diff --git a/shell/ash.c b/shell/ash.c
index a284b084d..507d4eb64 100644
--- a/shell/ash.c
+++ b/shell/ash.c
@@ -224,6 +224,14 @@
 #define    BASH_XTRACEFD        ENABLE_ASH_BASH_COMPAT
 #define    BASH_READ_D          ENABLE_ASH_BASH_COMPAT
 #define IF_BASH_READ_D              IF_ASH_BASH_COMPAT
+/* <(...) and >(...) */
+#if HAVE_DEV_FD
+# define    BASH_PROCESS_SUBST   ENABLE_ASH_BASH_COMPAT
+# define IF_BASH_PROCESS_SUBST       IF_ASH_BASH_COMPAT
+#else
+# define    BASH_PROCESS_SUBST 0
+# define IF_BASH_PROCESS_SUBST(...)
+#endif
 
 #if defined(__ANDROID_API__) && __ANDROID_API__ <= 24
 /* Bionic at least up to version 24 has no glob() */
@@ -714,6 +722,12 @@ out2str(const char *p)
 #define CTLENDARI    ((unsigned char)'\207')
 #define CTLQUOTEMARK ((unsigned char)'\210')
 #define CTL_LAST CTLQUOTEMARK
+#if BASH_PROCESS_SUBST
+# define CTLTOPROC    ((unsigned char)'\211')
+# define CTLFROMPROC  ((unsigned char)'\212')
+# undef CTL_LAST
+# define CTL_LAST CTLFROMPROC
+#endif
 
 /* variable substitution byte (follows CTLVAR) */
 #define VSTYPE  0x0f            /* type of variable substitution */
@@ -985,6 +999,10 @@ trace_puts_quoted(char *s)
 		case CTLESC: c = 'e'; goto backslash;
 		case CTLVAR: c = 'v'; goto backslash;
 		case CTLBACKQ: c = 'q'; goto backslash;
+#if BASH_PROCESS_SUBST
+		case CTLTOPROC: c = 'p'; goto backslash;
+		case CTLFROMPROC: c = 'P'; goto backslash;
+#endif
  backslash:
 			putc('\\', tracefile);
 			putc(c, tracefile);
@@ -1146,8 +1164,17 @@ sharg(union node *arg, FILE *fp)
 		case CTLENDVAR:
 			putc('}', fp);
 			break;
+#if BASH_PROCESS_SUBST
+		case CTLTOPROC:
+			putc('>', fp);
+			goto backq;
+		case CTLFROMPROC:
+			putc('<', fp);
+			goto backq;
+#endif
 		case CTLBACKQ:
 			putc('$', fp);
+ IF_BASH_PROCESS_SUBST(backq:)
 			putc('(', fp);
 			shtree(bqlist->n, -1, NULL, fp);
 			putc(')', fp);
@@ -3153,8 +3180,13 @@ static const uint8_t syntax_index_table[] ALIGN1 = {
 	/* 134 CTLARI       */ CCTL_CCTL_CCTL_CCTL,
 	/* 135 CTLENDARI    */ CCTL_CCTL_CCTL_CCTL,
 	/* 136 CTLQUOTEMARK */ CCTL_CCTL_CCTL_CCTL,
+#if BASH_PROCESS_SUBST
+	/* 137 CTLTOPROC    */ CCTL_CCTL_CCTL_CCTL,
+	/* 138 CTLFROMPROC  */ CCTL_CCTL_CCTL_CCTL,
+#else
 	/* 137      */ CWORD_CWORD_CWORD_CWORD,
 	/* 138      */ CWORD_CWORD_CWORD_CWORD,
+#endif
 	/* 139      */ CWORD_CWORD_CWORD_CWORD,
 	/* 140      */ CWORD_CWORD_CWORD_CWORD,
 	/* 141      */ CWORD_CWORD_CWORD_CWORD,
@@ -4723,9 +4755,24 @@ cmdputs(const char *s)
 			quoted >>= 1;
 			subtype = 0;
 			goto dostr;
+#if BASH_PROCESS_SUBST
+		case CTLBACKQ:
+			c = '$';
+			str = "(...)";
+			break;
+		case CTLTOPROC:
+			c = '>';
+			str = "(...)";
+			break;
+		case CTLFROMPROC:
+			c = '<';
+			str = "(...)";
+			break;
+#else
 		case CTLBACKQ:
 			str = "$(...)";
 			goto dostr;
+#endif
 #if ENABLE_FEATURE_SH_MATH
 		case CTLARI:
 			str = "$((";
@@ -5769,14 +5816,29 @@ redirectsafe(union node *redir, int flags)
 	return err;
 }
 
-static struct redirtab*
+#if BASH_PROCESS_SUBST
+static void
+pushfd(int fd)
+{
+	struct redirtab *sv;
+
+	sv = ckzalloc(sizeof(*sv) + sizeof(sv->two_fd[0]));
+	sv->pair_count = 1;
+	sv->two_fd[0].orig_fd = fd;
+	sv->two_fd[0].moved_to = CLOSED;
+	sv->next = redirlist;
+	redirlist = sv;
+}
+#endif
+
+static void
 pushredir(union node *redir)
 {
 	struct redirtab *sv;
 	int i;
 
 	if (!redir)
-		return redirlist;
+		return;
 
 	i = 0;
 	do {
@@ -5794,7 +5856,6 @@ pushredir(union node *redir)
 		sv->two_fd[i].orig_fd = sv->two_fd[i].moved_to = EMPTY;
 	sv->next = redirlist;
 	redirlist = sv;
-	return sv->next;
 }
 
 /*
@@ -6403,10 +6464,20 @@ evaltreenr(union node *n, int flags)
 }
 
 static void FAST_FUNC
-evalbackcmd(union node *n, struct backcmd *result)
+evalbackcmd(union node *n, struct backcmd *result
+				IF_BASH_PROCESS_SUBST(, int ctl))
 {
 	int pip[2];
 	struct job *jp;
+#if BASH_PROCESS_SUBST
+	/* determine end of pipe used by parent (ip) and child (ic) */
+	const int ip = (ctl == CTLTOPROC);
+	const int ic = !(ctl == CTLTOPROC);
+#else
+	const int ctl = CTLBACKQ;
+	const int ip = 0;
+	const int ic = 1;
+#endif
 
 	result->fd = -1;
 	result->buf = NULL;
@@ -6418,15 +6489,17 @@ evalbackcmd(union node *n, struct backcmd *result)
 
 	if (pipe(pip) < 0)
 		ash_msg_and_raise_perror("can't create pipe");
-	jp = makejob(/*n,*/ 1);
-	if (forkshell(jp, n, FORK_NOJOB) == 0) {
+	/* process substitution uses NULL job/node, like openhere() */
+	jp = (ctl == CTLBACKQ) ? makejob(/*n,*/ 1) : NULL;
+	if (forkshell(jp, (ctl == CTLBACKQ) ? n : NULL, FORK_NOJOB) == 0) {
 		/* child */
 		FORCE_INT_ON;
-		close(pip[0]);
-		if (pip[1] != 1) {
-			/*close(1);*/
-			dup2_or_raise(pip[1], 1);
-			close(pip[1]);
+		close(pip[ip]);
+		/* ic is index of child end of pipe *and* fd to connect it to */
+		if (pip[ic] != ic) {
+			/*close(ic);*/
+			dup2_or_raise(pip[ic], ic);
+			close(pip[ic]);
 		}
 /* TODO: eflag clearing makes the following not abort:
  *  ash -c 'set -e; z=$(false;echo foo); echo $z'
@@ -6442,8 +6515,18 @@ evalbackcmd(union node *n, struct backcmd *result)
 		/* NOTREACHED */
 	}
 	/* parent */
-	close(pip[1]);
-	result->fd = pip[0];
+#if BASH_PROCESS_SUBST
+	if (ctl != CTLBACKQ) {
+		int fd = fcntl(pip[ip], F_DUPFD, 64);
+		if (fd > 0) {
+			close(pip[ip]);
+			pip[ip] = fd;
+		}
+		pushfd(pip[ip]);
+	}
+#endif
+	close(pip[ic]);
+	result->fd = pip[ip];
 	result->jp = jp;
 
  out:
@@ -6455,8 +6538,11 @@ evalbackcmd(union node *n, struct backcmd *result)
  * Expand stuff in backwards quotes.
  */
 static void
-expbackq(union node *cmd, int flag)
+expbackq(union node *cmd, int flag IF_BASH_PROCESS_SUBST(, int ctl))
 {
+#if !BASH_PROCESS_SUBST
+	const int ctl = CTLBACKQ;
+#endif
 	struct backcmd in;
 	int i;
 	char buf[128];
@@ -6469,29 +6555,34 @@ expbackq(union node *cmd, int flag)
 	INT_OFF;
 	startloc = expdest - (char *)stackblock();
 	pushstackmark(&smark, startloc);
-	evalbackcmd(cmd, &in);
+	evalbackcmd(cmd, &in IF_BASH_PROCESS_SUBST(, ctl));
 	popstackmark(&smark);
 
-	p = in.buf;
-	i = in.nleft;
-	if (i == 0)
-		goto read;
-	for (;;) {
-		memtodest(p, i, syntax, flag & QUOTES_ESC);
+	if (ctl == CTLBACKQ) {
+		p = in.buf;
+		i = in.nleft;
+		if (i == 0)
+			goto read;
+		for (;;) {
+			memtodest(p, i, syntax, flag & QUOTES_ESC);
  read:
-		if (in.fd < 0)
-			break;
-		i = nonblock_immune_read(in.fd, buf, sizeof(buf));
-		TRACE(("expbackq: read returns %d\n", i));
-		if (i <= 0)
-			break;
-		p = buf;
-	}
+			if (in.fd < 0)
+				break;
+			i = nonblock_immune_read(in.fd, buf, sizeof(buf));
+			TRACE(("expbackq: read returns %d\n", i));
+			if (i <= 0)
+				break;
+			p = buf;
+		}
 
-	free(in.buf);
-	if (in.fd >= 0) {
-		close(in.fd);
-		back_exitstatus = waitforjob(in.jp);
+		free(in.buf);
+		if (in.fd >= 0) {
+			close(in.fd);
+			back_exitstatus = waitforjob(in.jp);
+		}
+	} else {
+		sprintf(buf, DEV_FD_PREFIX"%d", in.fd);
+		strtodest(buf, BASESYNTAX, 0);
 	}
 	INT_ON;
 
@@ -6587,6 +6678,10 @@ argstr(char *p, int flags)
 		CTLESC,
 		CTLVAR,
 		CTLBACKQ,
+#if BASH_PROCESS_SUBST
+		CTLTOPROC,
+		CTLFROMPROC,
+#endif
 #if ENABLE_FEATURE_SH_MATH
 		CTLENDARI,
 #endif
@@ -6689,8 +6784,12 @@ argstr(char *p, int flags)
 			p = evalvar(p, flags | inquotes);
 			TRACE(("argstr: evalvar:'%s'\n", (char *)stackblock()));
 			goto start;
+#if BASH_PROCESS_SUBST
+		case CTLTOPROC:
+		case CTLFROMPROC:
+#endif
 		case CTLBACKQ:
-			expbackq(argbackq->n, flags | inquotes);
+			expbackq(argbackq->n, flags | inquotes IF_BASH_PROCESS_SUBST(, c));
 			argbackq = argbackq->next;
 			goto start;
 #if ENABLE_FEATURE_SH_MATH
@@ -7433,7 +7532,8 @@ evalvar(char *p, int flag)
 			unsigned char c = *p++;
 			if (c == CTLESC)
 				p++;
-			else if (c == CTLBACKQ) {
+			else if (c == CTLBACKQ
+				IF_BASH_PROCESS_SUBST(|| c == CTLTOPROC || c == CTLFROMPROC)) {
 				if (varlen >= 0)
 					argbackq = argbackq->next;
 			} else if (c == CTLVAR) {
@@ -9995,6 +10095,7 @@ evalcommand(union node *cmd, int flags)
 	setstackmark(&smark);
 	localvar_stop = pushlocalvars();
 	file_stop = g_parsefile;
+	redir_stop = redirlist;
 	back_exitstatus = 0;
 
 	cmdentry.cmdtype = CMDBUILTIN;
@@ -10040,7 +10141,7 @@ evalcommand(union node *cmd, int flags)
 		lastarg = nargv[-1];
 
 	expredir(cmd->ncmd.redirect);
-	redir_stop = pushredir(cmd->ncmd.redirect);
+	pushredir(cmd->ncmd.redirect);
 	preverrout_fd = 2;
 	if (BASH_XTRACEFD && xflag) {
 		/* NB: bash closes fd == $BASH_XTRACEFD when it is changed.
@@ -12005,11 +12106,13 @@ realeofmark(const char *eofmark)
  * using goto's to implement the subroutine linkage.  The following macros
  * will run code that appears at the end of readtoken1.
  */
+enum { OLD, NEW, PSUB };
 #define CHECKEND()      {goto checkend; checkend_return:;}
 #define PARSEREDIR()    {goto parseredir; parseredir_return:;}
 #define PARSESUB()      {goto parsesub; parsesub_return:;}
-#define PARSEBACKQOLD() {oldstyle = 1; goto parsebackq; parsebackq_oldreturn:;}
-#define PARSEBACKQNEW() {oldstyle = 0; goto parsebackq; parsebackq_newreturn:;}
+#define PARSEBACKQOLD() {style = OLD; goto parsebackq; parsebackq_oldreturn:;}
+#define PARSEBACKQNEW() {style = NEW; goto parsebackq; parsebackq_newreturn:;}
+#define PARSEPROCSUB()  {style = PSUB; goto parsebackq; parsebackq_psreturn:;}
 #define PARSEARITH()    {goto parsearith; parsearith_return:;}
 static int
 readtoken1(int c, int syntax, char *eofmark, int striptabs)
@@ -12020,7 +12123,7 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
 	size_t len;
 	struct nodelist *bqlist;
 	smallint quotef;
-	smallint oldstyle;
+	smallint style;
 	smallint pssyntax;   /* we are expanding a prompt string */
 	IF_BASH_DOLLAR_SQUOTE(smallint bash_dollar_squote = 0;)
 	/* syntax stack */
@@ -12203,6 +12306,15 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
 						c = 0x100 + '>'; /* flag &> */
 					pungetc();
 				}
+#endif
+#if BASH_PROCESS_SUBST
+				if (c == '<' || c == '>') {
+					if (pgetc() == '(') {
+						PARSEPROCSUB();
+						break;
+					}
+					pungetc();
+				}
 #endif
 				goto endword;   /* exit outer loop */
 			}
@@ -12591,7 +12703,7 @@ parsebackq: {
 		memcpy(str, stackblock(), savelen);
 	}
 
-	if (oldstyle) {
+	if (style == OLD) {
 		/* We must read until the closing backquote, giving special
 		 * treatment to some slashes, and then push the string and
 		 * reread it as input, interpreting it normally.
@@ -12649,20 +12761,20 @@ parsebackq: {
 	*nlpp = stzalloc(sizeof(**nlpp));
 	/* (*nlpp)->next = NULL; - stzalloc did it */
 
-	if (oldstyle) {
+	if (style == OLD) {
 		saveprompt = doprompt;
 		doprompt = 0;
 	}
 
 	n = list(2);
 
-	if (oldstyle)
+	if (style == OLD)
 		doprompt = saveprompt;
 	else if (readtoken() != TRP)
 		raise_error_unexpected_syntax(TRP);
 
 	(*nlpp)->n = n;
-	if (oldstyle) {
+	if (style == OLD) {
 		/*
 		 * Start reading from old file again, ignoring any pushed back
 		 * tokens left from the backquote parsing
@@ -12677,9 +12789,18 @@ parsebackq: {
 		memcpy(out, str, savelen);
 		STADJUST(savelen, out);
 	}
-	USTPUTC(CTLBACKQ, out);
-	if (oldstyle)
+#if BASH_PROCESS_SUBST
+	if (style == PSUB)
+		USTPUTC(c == '<' ? CTLFROMPROC : CTLTOPROC, out);
+	else
+#endif
+		USTPUTC(CTLBACKQ, out);
+	if (style == OLD)
 		goto parsebackq_oldreturn;
+#if BASH_PROCESS_SUBST
+	else if (style == PSUB)
+		goto parsebackq_psreturn;
+#endif
 	goto parsebackq_newreturn;
 }
 
@@ -13115,6 +13236,9 @@ cmdloop(int top)
 #if JOBS
 		if (doing_jobctl)
 			showjobs(SHOW_CHANGED|SHOW_STDERR);
+#endif
+#if BASH_PROCESS_SUBST
+		unwindredir(NULL);
 #endif
 		inter = 0;
 		if (iflag && top) {
diff --git a/shell/ash_test/ash-psubst/bash_procsub.right b/shell/ash_test/ash-psubst/bash_procsub.right
new file mode 100644
index 000000000..aa16a96be
--- /dev/null
+++ b/shell/ash_test/ash-psubst/bash_procsub.right
@@ -0,0 +1,9 @@
+hello 1
+hello 2
+hello 3
+<(echo "hello 0")
+hello 4
+HI THERE
+hello error
+hello error
+hello stderr
diff --git a/shell/ash_test/ash-psubst/bash_procsub.tests b/shell/ash_test/ash-psubst/bash_procsub.tests
new file mode 100755
index 000000000..63b836782
--- /dev/null
+++ b/shell/ash_test/ash-psubst/bash_procsub.tests
@@ -0,0 +1,33 @@
+# simplest case
+cat <(echo "hello 1")
+
+# can have more than one
+cat <(echo "hello 2") <(echo "hello 3")
+
+# doesn't work in quotes
+echo "<(echo \"hello 0\")"
+
+# process substitution can be nested inside command substitution
+echo $(cat <(echo "hello 4"))
+
+# example from http://wiki.bash-hackers.org/syntax/expansion/proc_subst
+# process substitutions can be passed to a function as parameters or
+# variables
+f() {
+	cat "$1" >"$x"
+}
+x=>(tr '[:lower:]' '[:upper:]') f <(echo 'hi there')
+
+# process substitution can be combined with redirection on exec
+rm -f err
+# save stderr
+exec 4>&2
+# copy stderr to a file
+exec 2> >(tee err)
+echo "hello error" >&2
+sync
+# restore stderr
+exec 2>&4
+cat err
+rm -f err
+echo "hello stderr" >&2
-- 
2.20.1



More information about the busybox mailing list