[PATCH] ash: add process substitution in bash-compatibility mode

Ron Yorston rmy at pobox.com
Fri Nov 23 13:09:47 UTC 2018


Process substitution is a Korn shell feature that's also available
in bash and some other shells.  This patch implements process
substitution in ash when ASH_BASH_COMPAT is enabled.

function                                             old     new   delta
argstr                                              1313    1538    +225
readtoken1                                          3265    3366    +101
evaltree                                             606     686     +80
close_new_fds                                          -      70     +70
strtodest                                              -      56     +56
evalcommand                                         1724    1756     +32
.rodata                                           168516  168544     +28
cmdputs                                              393     418     +25
evalvar                                              697     714     +17
cmdloop                                              404     420     +16
ash_main                                            1346    1352      +6
static.spclchars                                       9      11      +2
varvalue                                             669     626     -43
------------------------------------------------------------------------------
(add/remove: 2/0 grow/shrink: 10/1 up/down: 658/-43)          Total: 615 bytes

Signed-off-by: Ron Yorston <rmy at pobox.com>
---
 include/platform.h |   2 +
 shell/ash.c        | 302 ++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 262 insertions(+), 42 deletions(-)

diff --git a/include/platform.h b/include/platform.h
index c365d5c8c..61d58961f 100644
--- a/include/platform.h
+++ b/include/platform.h
@@ -412,6 +412,8 @@ typedef unsigned smalluint;
 #define HAVE_NET_ETHERNET_H 1
 #define HAVE_SYS_STATFS_H 1
 #define HAVE_PRINTF_PERCENTM 1
+#define HAVE_DEV_FD 1
+#define DEV_FD_PREFIX "/dev/fd/"
 
 #if defined(__UCLIBC__)
 # if UCLIBC_VERSION < KERNEL_VERSION(0, 9, 32)
diff --git a/shell/ash.c b/shell/ash.c
index 04e4006c8..d6e17ebeb 100644
--- a/shell/ash.c
+++ b/shell/ash.c
@@ -238,6 +238,14 @@
 #define    BASH_XTRACEFD        ENABLE_ASH_BASH_COMPAT
 #define    BASH_READ_D          ENABLE_ASH_BASH_COMPAT
 #define IF_BASH_READ_D              IF_ASH_BASH_COMPAT
+/* <(...) and >(...) */
+#if HAVE_DEV_FD
+# define    BASH_PROCESS_SUBST   ENABLE_ASH_BASH_COMPAT
+# define IF_BASH_PROCESS_SUBST       IF_ASH_BASH_COMPAT
+#else
+# define    BASH_PROCESS_SUBST 0
+# define IF_BASH_PROCESS_SUBST(...)
+#endif
 
 #if defined(__ANDROID_API__) && __ANDROID_API__ <= 24
 /* Bionic at least up to version 24 has no glob() */
@@ -379,7 +387,11 @@ struct globals_misc {
 	int shlvl;
 #define rootshell (!shlvl)
 	int errlinno;
-
+#if BASH_PROCESS_SUBST
+	uint16_t *psfds;        /* for each fd, level it was opened or 0 */
+	int psmaxfds;           /* current size of fds array */
+	uint16_t pslevel;       /* current nesting level */
+#endif
 	char *minusc;  /* argument to -c option */
 
 	char *curdir; // = nullstr;     /* current working directory */
@@ -458,6 +470,9 @@ extern struct globals_misc *BB_GLOBAL_CONST ash_ptr_to_globals_misc;
 #define rootpid     (G_misc.rootpid    )
 #define shlvl       (G_misc.shlvl      )
 #define errlinno    (G_misc.errlinno   )
+#define psfds       (G_misc.psfds      )
+#define psmaxfds    (G_misc.psmaxfds   )
+#define pslevel     (G_misc.pslevel    )
 #define minusc      (G_misc.minusc     )
 #define curdir      (G_misc.curdir     )
 #define physdir     (G_misc.physdir    )
@@ -482,6 +497,7 @@ extern struct globals_misc *BB_GLOBAL_CONST ash_ptr_to_globals_misc;
 	barrier(); \
 	curdir = nullstr; \
 	physdir = nullstr; \
+	IF_BASH_PROCESS_SUBST(pslevel = 1;) \
 	trap_ptr = trap; \
 } while (0)
 
@@ -728,6 +744,12 @@ out2str(const char *p)
 #define CTLENDARI    ((unsigned char)'\207')
 #define CTLQUOTEMARK ((unsigned char)'\210')
 #define CTL_LAST CTLQUOTEMARK
+#if BASH_PROCESS_SUBST
+# define CTLTOPROC    ((unsigned char)'\211')
+# define CTLFROMPROC  ((unsigned char)'\212')
+# undef CTL_LAST
+# define CTL_LAST CTLFROMPROC
+#endif
 
 /* variable substitution byte (follows CTLVAR) */
 #define VSTYPE  0x0f            /* type of variable substitution */
@@ -999,6 +1021,10 @@ trace_puts_quoted(char *s)
 		case CTLESC: c = 'e'; goto backslash;
 		case CTLVAR: c = 'v'; goto backslash;
 		case CTLBACKQ: c = 'q'; goto backslash;
+#if BASH_PROCESS_SUBST
+		case CTLTOPROC: c = 'p'; goto backslash;
+		case CTLFROMPROC: c = 'P'; goto backslash;
+#endif
  backslash:
 			putc('\\', tracefile);
 			putc(c, tracefile);
@@ -1160,8 +1186,17 @@ sharg(union node *arg, FILE *fp)
 		case CTLENDVAR:
 			putc('}', fp);
 			break;
+#if BASH_PROCESS_SUBST
+		case CTLTOPROC:
+			putc('>', fp);
+			goto backq;
+		case CTLFROMPROC:
+			putc('<', fp);
+			goto backq;
+#endif
 		case CTLBACKQ:
 			putc('$', fp);
+ IF_BASH_PROCESS_SUBST(backq:)
 			putc('(', fp);
 			shtree(bqlist->n, -1, NULL, fp);
 			putc(')', fp);
@@ -3167,8 +3202,13 @@ static const uint8_t syntax_index_table[] ALIGN1 = {
 	/* 134 CTLARI       */ CCTL_CCTL_CCTL_CCTL,
 	/* 135 CTLENDARI    */ CCTL_CCTL_CCTL_CCTL,
 	/* 136 CTLQUOTEMARK */ CCTL_CCTL_CCTL_CCTL,
+#if BASH_PROCESS_SUBST
+	/* 137 CTLTOPROC    */ CCTL_CCTL_CCTL_CCTL,
+	/* 138 CTLFROMPROC  */ CCTL_CCTL_CCTL_CCTL,
+#else
 	/* 137      */ CWORD_CWORD_CWORD_CWORD,
 	/* 138      */ CWORD_CWORD_CWORD_CWORD,
+#endif
 	/* 139      */ CWORD_CWORD_CWORD_CWORD,
 	/* 140      */ CWORD_CWORD_CWORD_CWORD,
 	/* 141      */ CWORD_CWORD_CWORD_CWORD,
@@ -4737,9 +4777,24 @@ cmdputs(const char *s)
 			quoted >>= 1;
 			subtype = 0;
 			goto dostr;
+#if !BASH_PROCESS_SUBST
 		case CTLBACKQ:
 			str = "$(...)";
 			goto dostr;
+#else
+		case CTLBACKQ:
+			c = '$';
+			str = "(...)";
+			break;
+		case CTLTOPROC:
+			c = '>';
+			str = "(...)";
+			break;
+		case CTLFROMPROC:
+			c = '<';
+			str = "(...)";
+			break;
+#endif
 #if ENABLE_FEATURE_SH_MATH
 		case CTLARI:
 			str = "$((";
@@ -6384,6 +6439,75 @@ exptilde(char *startp, char *p, int flags)
 	return startp;
 }
 
+#if BASH_PROCESS_SUBST
+static void
+add_fd_to_list(int fd)
+{
+	int old_maxfds;
+
+	if (fd >= psmaxfds) {
+		old_maxfds = psmaxfds;
+		psmaxfds = fd + 32;
+		psfds = (uint16_t *)xrealloc(psfds, sizeof(uint16_t)*psmaxfds);
+		memset(psfds+old_maxfds, 0, sizeof(uint16_t)*(psmaxfds-old_maxfds));
+	}
+
+	psfds[fd] = pslevel;
+}
+
+static void
+close_new_fds(uint16_t level)
+{
+	int fd;
+
+	for (fd = 0; fd < psmaxfds; ++fd) {
+		if (psfds[fd] >= level) {
+			close(fd);
+			psfds[fd] = 0;
+		}
+	}
+}
+
+static void
+close_all_fds(void)
+{
+	close_new_fds(1);
+	pslevel = 1;
+}
+
+# if 1
+static uint16_t
+next_pslevel(void)
+{
+	if (pslevel == UINT16_MAX)
+		ash_msg_and_raise_error(bb_msg_memory_exhausted);
+
+	return ++pslevel;
+}
+# else
+/*
+ * Deeply recursive code might run out of levels to track file descriptors
+ * even if it isn't using process substitution.  The alternative code below
+ * increases pslevel only when necessary but costs about 50 bytes.
+ */
+static uint16_t
+next_pslevel(void)
+{
+	int fd;
+
+	if (pslevel == UINT16_MAX)
+		ash_msg_and_raise_error(bb_msg_memory_exhausted);
+
+	for (fd = 0; fd < psmaxfds; ++fd) {
+		if (psfds[fd] == pslevel)
+			return ++pslevel;
+	}
+
+	return pslevel;
+}
+# endif
+#endif
+
 /*
  * Execute a command inside back quotes.  If it's a builtin command, we
  * want to save its output in a block obtained from malloc.  Otherwise
@@ -6417,10 +6541,20 @@ evaltreenr(union node *n, int flags)
 }
 
 static void FAST_FUNC
-evalbackcmd(union node *n, struct backcmd *result)
+evalbackcmd(union node *n, struct backcmd *result
+				IF_BASH_PROCESS_SUBST(, int ctl))
 {
 	int pip[2];
 	struct job *jp;
+#if BASH_PROCESS_SUBST
+	/* determine end of pipe used by parent (ip) and child (ic) */
+	const int ip = (ctl == CTLTOPROC);
+	const int ic = !(ctl == CTLTOPROC);
+#else
+	const int ctl = CTLBACKQ;
+	const int ip = 0;
+	const int ic = 1;
+#endif
 
 	result->fd = -1;
 	result->buf = NULL;
@@ -6432,15 +6566,17 @@ evalbackcmd(union node *n, struct backcmd *result)
 
 	if (pipe(pip) < 0)
 		ash_msg_and_raise_perror("can't create pipe");
-	jp = makejob(/*n,*/ 1);
-	if (forkshell(jp, n, FORK_NOJOB) == 0) {
+	/* process substitution uses NULL job/node, like openhere() */
+	jp = (ctl == CTLBACKQ) ? makejob(/*n,*/ 1) : NULL;
+	if (forkshell(jp, (ctl == CTLBACKQ) ? n : NULL, FORK_NOJOB) == 0) {
 		/* child */
 		FORCE_INT_ON;
-		close(pip[0]);
-		if (pip[1] != 1) {
-			/*close(1);*/
-			dup2_or_raise(pip[1], 1);
-			close(pip[1]);
+		close(pip[ip]);
+		/* ic is index of child end of pipe *and* fd to connect it to */
+		if (pip[ic] != ic) {
+			/*close(ic);*/
+			dup2_or_raise(pip[ic], ic);
+			close(pip[ic]);
 		}
 /* TODO: eflag clearing makes the following not abort:
  *  ash -c 'set -e; z=$(false;echo foo); echo $z'
@@ -6456,8 +6592,18 @@ evalbackcmd(union node *n, struct backcmd *result)
 		/* NOTREACHED */
 	}
 	/* parent */
-	close(pip[1]);
-	result->fd = pip[0];
+#if BASH_PROCESS_SUBST
+	if (ctl != CTLBACKQ) {
+		int fd = fcntl(pip[ip], F_DUPFD, 64);
+		if (fd > 0) {
+			close(pip[ip]);
+			pip[ip] = fd;
+		}
+		add_fd_to_list(pip[ip]);
+	}
+#endif
+	close(pip[ic]);
+	result->fd = pip[ip];
 	result->jp = jp;
 
  out:
@@ -6469,8 +6615,11 @@ evalbackcmd(union node *n, struct backcmd *result)
  * Expand stuff in backwards quotes.
  */
 static void
-expbackq(union node *cmd, int flag)
+expbackq(union node *cmd, int flag IF_BASH_PROCESS_SUBST(, int ctl))
 {
+#if !BASH_PROCESS_SUBST
+	const int ctl = CTLBACKQ;
+#endif
 	struct backcmd in;
 	int i;
 	char buf[128];
@@ -6483,29 +6632,34 @@ expbackq(union node *cmd, int flag)
 	INT_OFF;
 	startloc = expdest - (char *)stackblock();
 	pushstackmark(&smark, startloc);
-	evalbackcmd(cmd, &in);
+	evalbackcmd(cmd, &in IF_BASH_PROCESS_SUBST(, ctl));
 	popstackmark(&smark);
 
-	p = in.buf;
-	i = in.nleft;
-	if (i == 0)
-		goto read;
-	for (;;) {
-		memtodest(p, i, syntax, flag & QUOTES_ESC);
+	if (ctl == CTLBACKQ) {
+		p = in.buf;
+		i = in.nleft;
+		if (i == 0)
+			goto read;
+		for (;;) {
+			memtodest(p, i, syntax, flag & QUOTES_ESC);
  read:
-		if (in.fd < 0)
-			break;
-		i = nonblock_immune_read(in.fd, buf, sizeof(buf));
-		TRACE(("expbackq: read returns %d\n", i));
-		if (i <= 0)
-			break;
-		p = buf;
-	}
+			if (in.fd < 0)
+				break;
+			i = nonblock_immune_read(in.fd, buf, sizeof(buf));
+			TRACE(("expbackq: read returns %d\n", i));
+			if (i <= 0)
+				break;
+			p = buf;
+		}
 
-	free(in.buf);
-	if (in.fd >= 0) {
-		close(in.fd);
-		back_exitstatus = waitforjob(in.jp);
+		free(in.buf);
+		if (in.fd >= 0) {
+			close(in.fd);
+			back_exitstatus = waitforjob(in.jp);
+		}
+	} else {
+		sprintf(buf, DEV_FD_PREFIX"%d", in.fd);
+		strtodest(buf, BASESYNTAX, 0);
 	}
 	INT_ON;
 
@@ -6601,6 +6755,10 @@ argstr(char *p, int flags)
 		CTLESC,
 		CTLVAR,
 		CTLBACKQ,
+#if BASH_PROCESS_SUBST
+		CTLTOPROC,
+		CTLFROMPROC,
+#endif
 #if ENABLE_FEATURE_SH_MATH
 		CTLENDARI,
 #endif
@@ -6703,8 +6861,12 @@ argstr(char *p, int flags)
 			p = evalvar(p, flags | inquotes);
 			TRACE(("argstr: evalvar:'%s'\n", (char *)stackblock()));
 			goto start;
+#if BASH_PROCESS_SUBST
+		case CTLTOPROC:
+		case CTLFROMPROC:
+#endif
 		case CTLBACKQ:
-			expbackq(argbackq->n, flags | inquotes);
+			expbackq(argbackq->n, flags | inquotes IF_BASH_PROCESS_SUBST(, c));
 			argbackq = argbackq->next;
 			goto start;
 #if ENABLE_FEATURE_SH_MATH
@@ -7447,7 +7609,8 @@ evalvar(char *p, int flag)
 			unsigned char c = *p++;
 			if (c == CTLESC)
 				p++;
-			else if (c == CTLBACKQ) {
+			else if (c == CTLBACKQ
+				IF_BASH_PROCESS_SUBST(|| c == CTLTOPROC || c == CTLFROMPROC)) {
 				if (varlen >= 0)
 					argbackq = argbackq->next;
 			} else if (c == CTLVAR) {
@@ -9049,6 +9212,10 @@ evaltree(union node *n, int flags)
 	int checkexit = 0;
 	int (*evalfn)(union node *, int);
 	int status = 0;
+#if BASH_PROCESS_SUBST
+	uint16_t old_level;
+	uint16_t level = 0;
+#endif
 
 	if (n == NULL) {
 		TRACE(("evaltree(NULL) called\n"));
@@ -9058,6 +9225,14 @@ evaltree(union node *n, int flags)
 
 	dotrap();
 
+#if BASH_PROCESS_SUBST
+	/* If we're in a shell function record new file descriptors so we
+	 * can close them later. */
+	if (funcline != 0) {
+		old_level = pslevel;
+		level = next_pslevel();
+	}
+#endif
 	switch (n->type) {
 	default:
 #if DEBUG
@@ -9153,6 +9328,17 @@ evaltree(union node *n, int flags)
 		break;
 	}
  out:
+#if BASH_PROCESS_SUBST
+	/* If we don't close new fds here
+	 *   f() { while true; do cat <(echo hi); done }
+	 *   f
+	 * runs out of file descriptors.
+	 */
+	if (level != 0) {
+		close_new_fds(level);
+		pslevel = old_level;
+	}
+#endif
 	/* Order of checks below is important:
 	 * signal handlers trigger before exit caused by "set -e".
 	 */
@@ -10282,6 +10468,15 @@ evalcommand(union node *cmd, int flags)
 	unwindredir(redir_stop);
 	unwindfiles(file_stop);
 	unwindlocalvars(localvar_stop);
+#if BASH_PROCESS_SUBST
+	/* Only close fds if we aren't in a shell function.  If we don't
+	 * close fds here
+	 *    while true; do cat <(echo hi); done
+	 * runs out of file descriptors.
+	 */
+	if (funcline == 0)
+		close_all_fds();
+#endif
 	if (lastarg) {
 		/* dsl: I think this is intended to be used to support
 		 * '_' in 'vi' command mode during line editing...
@@ -12019,11 +12214,13 @@ realeofmark(const char *eofmark)
  * using goto's to implement the subroutine linkage.  The following macros
  * will run code that appears at the end of readtoken1.
  */
+enum { OLD, NEW, PSUB };
 #define CHECKEND()      {goto checkend; checkend_return:;}
 #define PARSEREDIR()    {goto parseredir; parseredir_return:;}
 #define PARSESUB()      {goto parsesub; parsesub_return:;}
-#define PARSEBACKQOLD() {oldstyle = 1; goto parsebackq; parsebackq_oldreturn:;}
-#define PARSEBACKQNEW() {oldstyle = 0; goto parsebackq; parsebackq_newreturn:;}
+#define PARSEBACKQOLD() {style = OLD; goto parsebackq; parsebackq_oldreturn:;}
+#define PARSEBACKQNEW() {style = NEW; goto parsebackq; parsebackq_newreturn:;}
+#define PARSEPROCSUB()  {style = PSUB; goto parsebackq; parsebackq_psreturn:;}
 #define PARSEARITH()    {goto parsearith; parsearith_return:;}
 static int
 readtoken1(int c, int syntax, char *eofmark, int striptabs)
@@ -12034,7 +12231,7 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
 	size_t len;
 	struct nodelist *bqlist;
 	smallint quotef;
-	smallint oldstyle;
+	smallint style;
 	smallint pssyntax;   /* we are expanding a prompt string */
 	IF_BASH_DOLLAR_SQUOTE(smallint bash_dollar_squote = 0;)
 	/* syntax stack */
@@ -12217,6 +12414,15 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
 						c = 0x100 + '>'; /* flag &> */
 					pungetc();
 				}
+#endif
+#if BASH_PROCESS_SUBST
+				if (c == '<' || c == '>') {
+					if (pgetc() == '(') {
+						PARSEPROCSUB();
+						break;
+					}
+					pungetc();
+				}
 #endif
 				goto endword;   /* exit outer loop */
 			}
@@ -12605,7 +12811,7 @@ parsebackq: {
 		memcpy(str, stackblock(), savelen);
 	}
 
-	if (oldstyle) {
+	if (style == OLD) {
 		/* We must read until the closing backquote, giving special
 		 * treatment to some slashes, and then push the string and
 		 * reread it as input, interpreting it normally.
@@ -12663,20 +12869,20 @@ parsebackq: {
 	*nlpp = stzalloc(sizeof(**nlpp));
 	/* (*nlpp)->next = NULL; - stzalloc did it */
 
-	if (oldstyle) {
+	if (style == OLD) {
 		saveprompt = doprompt;
 		doprompt = 0;
 	}
 
 	n = list(2);
 
-	if (oldstyle)
+	if (style == OLD)
 		doprompt = saveprompt;
 	else if (readtoken() != TRP)
 		raise_error_unexpected_syntax(TRP);
 
 	(*nlpp)->n = n;
-	if (oldstyle) {
+	if (style == OLD) {
 		/*
 		 * Start reading from old file again, ignoring any pushed back
 		 * tokens left from the backquote parsing
@@ -12691,9 +12897,18 @@ parsebackq: {
 		memcpy(out, str, savelen);
 		STADJUST(savelen, out);
 	}
-	USTPUTC(CTLBACKQ, out);
-	if (oldstyle)
+#if BASH_PROCESS_SUBST
+	if (style == PSUB)
+		USTPUTC(c == '<' ? CTLFROMPROC : CTLTOPROC, out);
+	else
+#endif
+		USTPUTC(CTLBACKQ, out);
+	if (style == OLD)
 		goto parsebackq_oldreturn;
+#if BASH_PROCESS_SUBST
+	else if (style == PSUB)
+		goto parsebackq_psreturn;
+#endif
 	goto parsebackq_newreturn;
 }
 
@@ -13129,6 +13344,9 @@ cmdloop(int top)
 #if JOBS
 		if (doing_jobctl)
 			showjobs(SHOW_CHANGED|SHOW_STDERR);
+#endif
+#if BASH_PROCESS_SUBST
+		close_all_fds();
 #endif
 		inter = 0;
 		if (iflag && top) {
-- 
2.19.1



More information about the busybox mailing list