[PATCH] ash: add process substitution in bash-compatibility mode
Ron Yorston
rmy at pobox.com
Fri Nov 23 13:09:47 UTC 2018
Process substitution is a Korn shell feature that's also available
in bash and some other shells. This patch implements process
substitution in ash when ASH_BASH_COMPAT is enabled.
function old new delta
argstr 1313 1538 +225
readtoken1 3265 3366 +101
evaltree 606 686 +80
close_new_fds - 70 +70
strtodest - 56 +56
evalcommand 1724 1756 +32
.rodata 168516 168544 +28
cmdputs 393 418 +25
evalvar 697 714 +17
cmdloop 404 420 +16
ash_main 1346 1352 +6
static.spclchars 9 11 +2
varvalue 669 626 -43
------------------------------------------------------------------------------
(add/remove: 2/0 grow/shrink: 10/1 up/down: 658/-43) Total: 615 bytes
Signed-off-by: Ron Yorston <rmy at pobox.com>
---
include/platform.h | 2 +
shell/ash.c | 302 ++++++++++++++++++++++++++++++++++++++-------
2 files changed, 262 insertions(+), 42 deletions(-)
diff --git a/include/platform.h b/include/platform.h
index c365d5c8c..61d58961f 100644
--- a/include/platform.h
+++ b/include/platform.h
@@ -412,6 +412,8 @@ typedef unsigned smalluint;
#define HAVE_NET_ETHERNET_H 1
#define HAVE_SYS_STATFS_H 1
#define HAVE_PRINTF_PERCENTM 1
+#define HAVE_DEV_FD 1
+#define DEV_FD_PREFIX "/dev/fd/"
#if defined(__UCLIBC__)
# if UCLIBC_VERSION < KERNEL_VERSION(0, 9, 32)
diff --git a/shell/ash.c b/shell/ash.c
index 04e4006c8..d6e17ebeb 100644
--- a/shell/ash.c
+++ b/shell/ash.c
@@ -238,6 +238,14 @@
#define BASH_XTRACEFD ENABLE_ASH_BASH_COMPAT
#define BASH_READ_D ENABLE_ASH_BASH_COMPAT
#define IF_BASH_READ_D IF_ASH_BASH_COMPAT
+/* <(...) and >(...) */
+#if HAVE_DEV_FD
+# define BASH_PROCESS_SUBST ENABLE_ASH_BASH_COMPAT
+# define IF_BASH_PROCESS_SUBST IF_ASH_BASH_COMPAT
+#else
+# define BASH_PROCESS_SUBST 0
+# define IF_BASH_PROCESS_SUBST(...)
+#endif
#if defined(__ANDROID_API__) && __ANDROID_API__ <= 24
/* Bionic at least up to version 24 has no glob() */
@@ -379,7 +387,11 @@ struct globals_misc {
int shlvl;
#define rootshell (!shlvl)
int errlinno;
-
+#if BASH_PROCESS_SUBST
+ uint16_t *psfds; /* for each fd, level it was opened or 0 */
+ int psmaxfds; /* current size of fds array */
+ uint16_t pslevel; /* current nesting level */
+#endif
char *minusc; /* argument to -c option */
char *curdir; // = nullstr; /* current working directory */
@@ -458,6 +470,9 @@ extern struct globals_misc *BB_GLOBAL_CONST ash_ptr_to_globals_misc;
#define rootpid (G_misc.rootpid )
#define shlvl (G_misc.shlvl )
#define errlinno (G_misc.errlinno )
+#define psfds (G_misc.psfds )
+#define psmaxfds (G_misc.psmaxfds )
+#define pslevel (G_misc.pslevel )
#define minusc (G_misc.minusc )
#define curdir (G_misc.curdir )
#define physdir (G_misc.physdir )
@@ -482,6 +497,7 @@ extern struct globals_misc *BB_GLOBAL_CONST ash_ptr_to_globals_misc;
barrier(); \
curdir = nullstr; \
physdir = nullstr; \
+ IF_BASH_PROCESS_SUBST(pslevel = 1;) \
trap_ptr = trap; \
} while (0)
@@ -728,6 +744,12 @@ out2str(const char *p)
#define CTLENDARI ((unsigned char)'\207')
#define CTLQUOTEMARK ((unsigned char)'\210')
#define CTL_LAST CTLQUOTEMARK
+#if BASH_PROCESS_SUBST
+# define CTLTOPROC ((unsigned char)'\211')
+# define CTLFROMPROC ((unsigned char)'\212')
+# undef CTL_LAST
+# define CTL_LAST CTLFROMPROC
+#endif
/* variable substitution byte (follows CTLVAR) */
#define VSTYPE 0x0f /* type of variable substitution */
@@ -999,6 +1021,10 @@ trace_puts_quoted(char *s)
case CTLESC: c = 'e'; goto backslash;
case CTLVAR: c = 'v'; goto backslash;
case CTLBACKQ: c = 'q'; goto backslash;
+#if BASH_PROCESS_SUBST
+ case CTLTOPROC: c = 'p'; goto backslash;
+ case CTLFROMPROC: c = 'P'; goto backslash;
+#endif
backslash:
putc('\\', tracefile);
putc(c, tracefile);
@@ -1160,8 +1186,17 @@ sharg(union node *arg, FILE *fp)
case CTLENDVAR:
putc('}', fp);
break;
+#if BASH_PROCESS_SUBST
+ case CTLTOPROC:
+ putc('>', fp);
+ goto backq;
+ case CTLFROMPROC:
+ putc('<', fp);
+ goto backq;
+#endif
case CTLBACKQ:
putc('$', fp);
+ IF_BASH_PROCESS_SUBST(backq:)
putc('(', fp);
shtree(bqlist->n, -1, NULL, fp);
putc(')', fp);
@@ -3167,8 +3202,13 @@ static const uint8_t syntax_index_table[] ALIGN1 = {
/* 134 CTLARI */ CCTL_CCTL_CCTL_CCTL,
/* 135 CTLENDARI */ CCTL_CCTL_CCTL_CCTL,
/* 136 CTLQUOTEMARK */ CCTL_CCTL_CCTL_CCTL,
+#if BASH_PROCESS_SUBST
+ /* 137 CTLTOPROC */ CCTL_CCTL_CCTL_CCTL,
+ /* 138 CTLFROMPROC */ CCTL_CCTL_CCTL_CCTL,
+#else
/* 137 */ CWORD_CWORD_CWORD_CWORD,
/* 138 */ CWORD_CWORD_CWORD_CWORD,
+#endif
/* 139 */ CWORD_CWORD_CWORD_CWORD,
/* 140 */ CWORD_CWORD_CWORD_CWORD,
/* 141 */ CWORD_CWORD_CWORD_CWORD,
@@ -4737,9 +4777,24 @@ cmdputs(const char *s)
quoted >>= 1;
subtype = 0;
goto dostr;
+#if !BASH_PROCESS_SUBST
case CTLBACKQ:
str = "$(...)";
goto dostr;
+#else
+ case CTLBACKQ:
+ c = '$';
+ str = "(...)";
+ break;
+ case CTLTOPROC:
+ c = '>';
+ str = "(...)";
+ break;
+ case CTLFROMPROC:
+ c = '<';
+ str = "(...)";
+ break;
+#endif
#if ENABLE_FEATURE_SH_MATH
case CTLARI:
str = "$((";
@@ -6384,6 +6439,75 @@ exptilde(char *startp, char *p, int flags)
return startp;
}
+#if BASH_PROCESS_SUBST
+static void
+add_fd_to_list(int fd)
+{
+ int old_maxfds;
+
+ if (fd >= psmaxfds) {
+ old_maxfds = psmaxfds;
+ psmaxfds = fd + 32;
+ psfds = (uint16_t *)xrealloc(psfds, sizeof(uint16_t)*psmaxfds);
+ memset(psfds+old_maxfds, 0, sizeof(uint16_t)*(psmaxfds-old_maxfds));
+ }
+
+ psfds[fd] = pslevel;
+}
+
+static void
+close_new_fds(uint16_t level)
+{
+ int fd;
+
+ for (fd = 0; fd < psmaxfds; ++fd) {
+ if (psfds[fd] >= level) {
+ close(fd);
+ psfds[fd] = 0;
+ }
+ }
+}
+
+static void
+close_all_fds(void)
+{
+ close_new_fds(1);
+ pslevel = 1;
+}
+
+# if 1
+static uint16_t
+next_pslevel(void)
+{
+ if (pslevel == UINT16_MAX)
+ ash_msg_and_raise_error(bb_msg_memory_exhausted);
+
+ return ++pslevel;
+}
+# else
+/*
+ * Deeply recursive code might run out of levels to track file descriptors
+ * even if it isn't using process substitution. The alternative code below
+ * increases pslevel only when necessary but costs about 50 bytes.
+ */
+static uint16_t
+next_pslevel(void)
+{
+ int fd;
+
+ if (pslevel == UINT16_MAX)
+ ash_msg_and_raise_error(bb_msg_memory_exhausted);
+
+ for (fd = 0; fd < psmaxfds; ++fd) {
+ if (psfds[fd] == pslevel)
+ return ++pslevel;
+ }
+
+ return pslevel;
+}
+# endif
+#endif
+
/*
* Execute a command inside back quotes. If it's a builtin command, we
* want to save its output in a block obtained from malloc. Otherwise
@@ -6417,10 +6541,20 @@ evaltreenr(union node *n, int flags)
}
static void FAST_FUNC
-evalbackcmd(union node *n, struct backcmd *result)
+evalbackcmd(union node *n, struct backcmd *result
+ IF_BASH_PROCESS_SUBST(, int ctl))
{
int pip[2];
struct job *jp;
+#if BASH_PROCESS_SUBST
+ /* determine end of pipe used by parent (ip) and child (ic) */
+ const int ip = (ctl == CTLTOPROC);
+ const int ic = !(ctl == CTLTOPROC);
+#else
+ const int ctl = CTLBACKQ;
+ const int ip = 0;
+ const int ic = 1;
+#endif
result->fd = -1;
result->buf = NULL;
@@ -6432,15 +6566,17 @@ evalbackcmd(union node *n, struct backcmd *result)
if (pipe(pip) < 0)
ash_msg_and_raise_perror("can't create pipe");
- jp = makejob(/*n,*/ 1);
- if (forkshell(jp, n, FORK_NOJOB) == 0) {
+ /* process substitution uses NULL job/node, like openhere() */
+ jp = (ctl == CTLBACKQ) ? makejob(/*n,*/ 1) : NULL;
+ if (forkshell(jp, (ctl == CTLBACKQ) ? n : NULL, FORK_NOJOB) == 0) {
/* child */
FORCE_INT_ON;
- close(pip[0]);
- if (pip[1] != 1) {
- /*close(1);*/
- dup2_or_raise(pip[1], 1);
- close(pip[1]);
+ close(pip[ip]);
+ /* ic is index of child end of pipe *and* fd to connect it to */
+ if (pip[ic] != ic) {
+ /*close(ic);*/
+ dup2_or_raise(pip[ic], ic);
+ close(pip[ic]);
}
/* TODO: eflag clearing makes the following not abort:
* ash -c 'set -e; z=$(false;echo foo); echo $z'
@@ -6456,8 +6592,18 @@ evalbackcmd(union node *n, struct backcmd *result)
/* NOTREACHED */
}
/* parent */
- close(pip[1]);
- result->fd = pip[0];
+#if BASH_PROCESS_SUBST
+ if (ctl != CTLBACKQ) {
+ int fd = fcntl(pip[ip], F_DUPFD, 64);
+ if (fd > 0) {
+ close(pip[ip]);
+ pip[ip] = fd;
+ }
+ add_fd_to_list(pip[ip]);
+ }
+#endif
+ close(pip[ic]);
+ result->fd = pip[ip];
result->jp = jp;
out:
@@ -6469,8 +6615,11 @@ evalbackcmd(union node *n, struct backcmd *result)
* Expand stuff in backwards quotes.
*/
static void
-expbackq(union node *cmd, int flag)
+expbackq(union node *cmd, int flag IF_BASH_PROCESS_SUBST(, int ctl))
{
+#if !BASH_PROCESS_SUBST
+ const int ctl = CTLBACKQ;
+#endif
struct backcmd in;
int i;
char buf[128];
@@ -6483,29 +6632,34 @@ expbackq(union node *cmd, int flag)
INT_OFF;
startloc = expdest - (char *)stackblock();
pushstackmark(&smark, startloc);
- evalbackcmd(cmd, &in);
+ evalbackcmd(cmd, &in IF_BASH_PROCESS_SUBST(, ctl));
popstackmark(&smark);
- p = in.buf;
- i = in.nleft;
- if (i == 0)
- goto read;
- for (;;) {
- memtodest(p, i, syntax, flag & QUOTES_ESC);
+ if (ctl == CTLBACKQ) {
+ p = in.buf;
+ i = in.nleft;
+ if (i == 0)
+ goto read;
+ for (;;) {
+ memtodest(p, i, syntax, flag & QUOTES_ESC);
read:
- if (in.fd < 0)
- break;
- i = nonblock_immune_read(in.fd, buf, sizeof(buf));
- TRACE(("expbackq: read returns %d\n", i));
- if (i <= 0)
- break;
- p = buf;
- }
+ if (in.fd < 0)
+ break;
+ i = nonblock_immune_read(in.fd, buf, sizeof(buf));
+ TRACE(("expbackq: read returns %d\n", i));
+ if (i <= 0)
+ break;
+ p = buf;
+ }
- free(in.buf);
- if (in.fd >= 0) {
- close(in.fd);
- back_exitstatus = waitforjob(in.jp);
+ free(in.buf);
+ if (in.fd >= 0) {
+ close(in.fd);
+ back_exitstatus = waitforjob(in.jp);
+ }
+ } else {
+ sprintf(buf, DEV_FD_PREFIX"%d", in.fd);
+ strtodest(buf, BASESYNTAX, 0);
}
INT_ON;
@@ -6601,6 +6755,10 @@ argstr(char *p, int flags)
CTLESC,
CTLVAR,
CTLBACKQ,
+#if BASH_PROCESS_SUBST
+ CTLTOPROC,
+ CTLFROMPROC,
+#endif
#if ENABLE_FEATURE_SH_MATH
CTLENDARI,
#endif
@@ -6703,8 +6861,12 @@ argstr(char *p, int flags)
p = evalvar(p, flags | inquotes);
TRACE(("argstr: evalvar:'%s'\n", (char *)stackblock()));
goto start;
+#if BASH_PROCESS_SUBST
+ case CTLTOPROC:
+ case CTLFROMPROC:
+#endif
case CTLBACKQ:
- expbackq(argbackq->n, flags | inquotes);
+ expbackq(argbackq->n, flags | inquotes IF_BASH_PROCESS_SUBST(, c));
argbackq = argbackq->next;
goto start;
#if ENABLE_FEATURE_SH_MATH
@@ -7447,7 +7609,8 @@ evalvar(char *p, int flag)
unsigned char c = *p++;
if (c == CTLESC)
p++;
- else if (c == CTLBACKQ) {
+ else if (c == CTLBACKQ
+ IF_BASH_PROCESS_SUBST(|| c == CTLTOPROC || c == CTLFROMPROC)) {
if (varlen >= 0)
argbackq = argbackq->next;
} else if (c == CTLVAR) {
@@ -9049,6 +9212,10 @@ evaltree(union node *n, int flags)
int checkexit = 0;
int (*evalfn)(union node *, int);
int status = 0;
+#if BASH_PROCESS_SUBST
+ uint16_t old_level;
+ uint16_t level = 0;
+#endif
if (n == NULL) {
TRACE(("evaltree(NULL) called\n"));
@@ -9058,6 +9225,14 @@ evaltree(union node *n, int flags)
dotrap();
+#if BASH_PROCESS_SUBST
+ /* If we're in a shell function record new file descriptors so we
+ * can close them later. */
+ if (funcline != 0) {
+ old_level = pslevel;
+ level = next_pslevel();
+ }
+#endif
switch (n->type) {
default:
#if DEBUG
@@ -9153,6 +9328,17 @@ evaltree(union node *n, int flags)
break;
}
out:
+#if BASH_PROCESS_SUBST
+ /* If we don't close new fds here
+ * f() { while true; do cat <(echo hi); done }
+ * f
+ * runs out of file descriptors.
+ */
+ if (level != 0) {
+ close_new_fds(level);
+ pslevel = old_level;
+ }
+#endif
/* Order of checks below is important:
* signal handlers trigger before exit caused by "set -e".
*/
@@ -10282,6 +10468,15 @@ evalcommand(union node *cmd, int flags)
unwindredir(redir_stop);
unwindfiles(file_stop);
unwindlocalvars(localvar_stop);
+#if BASH_PROCESS_SUBST
+ /* Only close fds if we aren't in a shell function. If we don't
+ * close fds here
+ * while true; do cat <(echo hi); done
+ * runs out of file descriptors.
+ */
+ if (funcline == 0)
+ close_all_fds();
+#endif
if (lastarg) {
/* dsl: I think this is intended to be used to support
* '_' in 'vi' command mode during line editing...
@@ -12019,11 +12214,13 @@ realeofmark(const char *eofmark)
* using goto's to implement the subroutine linkage. The following macros
* will run code that appears at the end of readtoken1.
*/
+enum { OLD, NEW, PSUB };
#define CHECKEND() {goto checkend; checkend_return:;}
#define PARSEREDIR() {goto parseredir; parseredir_return:;}
#define PARSESUB() {goto parsesub; parsesub_return:;}
-#define PARSEBACKQOLD() {oldstyle = 1; goto parsebackq; parsebackq_oldreturn:;}
-#define PARSEBACKQNEW() {oldstyle = 0; goto parsebackq; parsebackq_newreturn:;}
+#define PARSEBACKQOLD() {style = OLD; goto parsebackq; parsebackq_oldreturn:;}
+#define PARSEBACKQNEW() {style = NEW; goto parsebackq; parsebackq_newreturn:;}
+#define PARSEPROCSUB() {style = PSUB; goto parsebackq; parsebackq_psreturn:;}
#define PARSEARITH() {goto parsearith; parsearith_return:;}
static int
readtoken1(int c, int syntax, char *eofmark, int striptabs)
@@ -12034,7 +12231,7 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
size_t len;
struct nodelist *bqlist;
smallint quotef;
- smallint oldstyle;
+ smallint style;
smallint pssyntax; /* we are expanding a prompt string */
IF_BASH_DOLLAR_SQUOTE(smallint bash_dollar_squote = 0;)
/* syntax stack */
@@ -12217,6 +12414,15 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
c = 0x100 + '>'; /* flag &> */
pungetc();
}
+#endif
+#if BASH_PROCESS_SUBST
+ if (c == '<' || c == '>') {
+ if (pgetc() == '(') {
+ PARSEPROCSUB();
+ break;
+ }
+ pungetc();
+ }
#endif
goto endword; /* exit outer loop */
}
@@ -12605,7 +12811,7 @@ parsebackq: {
memcpy(str, stackblock(), savelen);
}
- if (oldstyle) {
+ if (style == OLD) {
/* We must read until the closing backquote, giving special
* treatment to some slashes, and then push the string and
* reread it as input, interpreting it normally.
@@ -12663,20 +12869,20 @@ parsebackq: {
*nlpp = stzalloc(sizeof(**nlpp));
/* (*nlpp)->next = NULL; - stzalloc did it */
- if (oldstyle) {
+ if (style == OLD) {
saveprompt = doprompt;
doprompt = 0;
}
n = list(2);
- if (oldstyle)
+ if (style == OLD)
doprompt = saveprompt;
else if (readtoken() != TRP)
raise_error_unexpected_syntax(TRP);
(*nlpp)->n = n;
- if (oldstyle) {
+ if (style == OLD) {
/*
* Start reading from old file again, ignoring any pushed back
* tokens left from the backquote parsing
@@ -12691,9 +12897,18 @@ parsebackq: {
memcpy(out, str, savelen);
STADJUST(savelen, out);
}
- USTPUTC(CTLBACKQ, out);
- if (oldstyle)
+#if BASH_PROCESS_SUBST
+ if (style == PSUB)
+ USTPUTC(c == '<' ? CTLFROMPROC : CTLTOPROC, out);
+ else
+#endif
+ USTPUTC(CTLBACKQ, out);
+ if (style == OLD)
goto parsebackq_oldreturn;
+#if BASH_PROCESS_SUBST
+ else if (style == PSUB)
+ goto parsebackq_psreturn;
+#endif
goto parsebackq_newreturn;
}
@@ -13129,6 +13344,9 @@ cmdloop(int top)
#if JOBS
if (doing_jobctl)
showjobs(SHOW_CHANGED|SHOW_STDERR);
+#endif
+#if BASH_PROCESS_SUBST
+ close_all_fds();
#endif
inter = 0;
if (iflag && top) {
--
2.19.1
More information about the busybox
mailing list