[PATCH v6] ash: add process substitution in bash-compatibility mode

Denys Vlasenko vda.linux at googlemail.com
Sat Jun 5 21:40:25 UTC 2021


Sorry for a very late reply.
Applied, thank you!

On Thu, Jul 23, 2020 at 9:37 AM Ron Yorston <rmy at pobox.com> wrote:
>
> Process substitution is a Korn shell feature that's also available
> in bash and some other shells.  This patch implements process
> substitution in ash when ASH_BASH_COMPAT is enabled.
>
> function                                             old     new   delta
> argstr                                              1386    1522    +136
> strtodest                                              -      52     +52
> readtoken1                                          3346    3392     +46
> .rodata                                           183206  183250     +44
> unwindredir                                            -      28     +28
> cmdloop                                              365     372      +7
> static.spclchars                                      10      12      +2
> cmdputs                                              380     367     -13
> exitreset                                             86      69     -17
> evalcommand                                         1754    1737     -17
> varvalue                                             675     634     -41
> ------------------------------------------------------------------------------
> (add/remove: 2/0 grow/shrink: 5/4 up/down: 315/-88)           Total: 227 bytes
>    text    data     bss     dec     hex filename
>  953967    4219    1904  960090   ea65a busybox_old
>  954192    4219    1904  960315   ea73b busybox_unstripped
>
> v2: Replace array of file descriptors with a linked list.
>     Include tests that were unaccountably omitted from v1.
> v3: Update linked list code to the intended version.
> v4: Change order of conditional code in cmdputs().
> v5: Use existing popredir() mechanism to manage file descriptors.
> v6: Rebase to latest version of BusyBox ash.  Reduce code churn.
>
> Signed-off-by: Ron Yorston <rmy at pobox.com>
> ---
>  include/platform.h                           |   2 +
>  shell/ash.c                                  | 160 +++++++++++++++++--
>  shell/ash_test/ash-psubst/bash_procsub.right |   9 ++
>  shell/ash_test/ash-psubst/bash_procsub.tests |  33 ++++
>  4 files changed, 187 insertions(+), 17 deletions(-)
>  create mode 100644 shell/ash_test/ash-psubst/bash_procsub.right
>  create mode 100755 shell/ash_test/ash-psubst/bash_procsub.tests
>
> diff --git a/include/platform.h b/include/platform.h
> index 43bb391bd..233cbe87c 100644
> --- a/include/platform.h
> +++ b/include/platform.h
> @@ -421,6 +421,8 @@ typedef unsigned smalluint;
>  #define HAVE_NET_ETHERNET_H 1
>  #define HAVE_SYS_STATFS_H 1
>  #define HAVE_PRINTF_PERCENTM 1
> +#define HAVE_DEV_FD 1
> +#define DEV_FD_PREFIX "/dev/fd/"
>
>  #if defined(__UCLIBC__)
>  # if UCLIBC_VERSION < KERNEL_VERSION(0, 9, 32)
> diff --git a/shell/ash.c b/shell/ash.c
> index ecb9b132b..72bf49896 100644
> --- a/shell/ash.c
> +++ b/shell/ash.c
> @@ -229,6 +229,14 @@
>  #define    BASH_READ_D          ENABLE_ASH_BASH_COMPAT
>  #define IF_BASH_READ_D              IF_ASH_BASH_COMPAT
>  #define    BASH_WAIT_N          ENABLE_ASH_BASH_COMPAT
> +/* <(...) and >(...) */
> +#if HAVE_DEV_FD
> +# define    BASH_PROCESS_SUBST   ENABLE_ASH_BASH_COMPAT
> +# define IF_BASH_PROCESS_SUBST       IF_ASH_BASH_COMPAT
> +#else
> +# define    BASH_PROCESS_SUBST 0
> +# define IF_BASH_PROCESS_SUBST(...)
> +#endif
>
>  #if defined(__ANDROID_API__) && __ANDROID_API__ <= 24
>  /* Bionic at least up to version 24 has no glob() */
> @@ -745,6 +753,12 @@ out2str(const char *p)
>  #define CTLENDARI    ((unsigned char)'\207')
>  #define CTLQUOTEMARK ((unsigned char)'\210')
>  #define CTL_LAST CTLQUOTEMARK
> +#if BASH_PROCESS_SUBST
> +# define CTLTOPROC    ((unsigned char)'\211')
> +# define CTLFROMPROC  ((unsigned char)'\212')
> +# undef CTL_LAST
> +# define CTL_LAST CTLFROMPROC
> +#endif
>
>  /* variable substitution byte (follows CTLVAR) */
>  #define VSTYPE  0x0f            /* type of variable substitution */
> @@ -1016,6 +1030,10 @@ trace_puts_quoted(char *s)
>                 case CTLESC: c = 'e'; goto backslash;
>                 case CTLVAR: c = 'v'; goto backslash;
>                 case CTLBACKQ: c = 'q'; goto backslash;
> +#if BASH_PROCESS_SUBST
> +               case CTLTOPROC: c = 'p'; goto backslash;
> +               case CTLFROMPROC: c = 'P'; goto backslash;
> +#endif
>   backslash:
>                         putc('\\', tracefile);
>                         putc(c, tracefile);
> @@ -1177,8 +1195,17 @@ sharg(union node *arg, FILE *fp)
>                 case CTLENDVAR:
>                         putc('}', fp);
>                         break;
> +#if BASH_PROCESS_SUBST
> +               case CTLTOPROC:
> +                       putc('>', fp);
> +                       goto backq;
> +               case CTLFROMPROC:
> +                       putc('<', fp);
> +                       goto backq;
> +#endif
>                 case CTLBACKQ:
>                         putc('$', fp);
> + IF_BASH_PROCESS_SUBST(backq:)
>                         putc('(', fp);
>                         shtree(bqlist->n, -1, NULL, fp);
>                         putc(')', fp);
> @@ -3230,8 +3257,13 @@ static const uint8_t syntax_index_table[] ALIGN1 = {
>         /* 134 CTLARI       */ CCTL_CCTL_CCTL_CCTL,
>         /* 135 CTLENDARI    */ CCTL_CCTL_CCTL_CCTL,
>         /* 136 CTLQUOTEMARK */ CCTL_CCTL_CCTL_CCTL,
> +#if BASH_PROCESS_SUBST
> +       /* 137 CTLTOPROC    */ CCTL_CCTL_CCTL_CCTL,
> +       /* 138 CTLFROMPROC  */ CCTL_CCTL_CCTL_CCTL,
> +#else
>         /* 137      */ CWORD_CWORD_CWORD_CWORD,
>         /* 138      */ CWORD_CWORD_CWORD_CWORD,
> +#endif
>         /* 139      */ CWORD_CWORD_CWORD_CWORD,
>         /* 140      */ CWORD_CWORD_CWORD_CWORD,
>         /* 141      */ CWORD_CWORD_CWORD_CWORD,
> @@ -4857,9 +4889,24 @@ cmdputs(const char *s)
>                         quoted >>= 1;
>                         subtype = 0;
>                         goto dostr;
> +#if BASH_PROCESS_SUBST
> +               case CTLBACKQ:
> +                       c = '$';
> +                       str = "(...)";
> +                       break;
> +               case CTLTOPROC:
> +                       c = '>';
> +                       str = "(...)";
> +                       break;
> +               case CTLFROMPROC:
> +                       c = '<';
> +                       str = "(...)";
> +                       break;
> +#else
>                 case CTLBACKQ:
>                         str = "$(...)";
>                         goto dostr;
> +#endif
>  #if ENABLE_FEATURE_SH_MATH
>                 case CTLARI:
>                         str = "$((";
> @@ -5899,6 +5946,21 @@ redirectsafe(union node *redir, int flags)
>         return err;
>  }
>
> +#if BASH_PROCESS_SUBST
> +static void
> +pushfd(int fd)
> +{
> +       struct redirtab *sv;
> +
> +       sv = ckzalloc(sizeof(*sv) + sizeof(sv->two_fd[0]));
> +       sv->pair_count = 1;
> +       sv->two_fd[0].orig_fd = fd;
> +       sv->two_fd[0].moved_to = CLOSED;
> +       sv->next = redirlist;
> +       redirlist = sv;
> +}
> +#endif
> +
>  static struct redirtab*
>  pushredir(union node *redir)
>  {
> @@ -6537,10 +6599,20 @@ evaltreenr(union node *n, int flags)
>  }
>
>  static void FAST_FUNC
> -evalbackcmd(union node *n, struct backcmd *result)
> +evalbackcmd(union node *n, struct backcmd *result
> +                               IF_BASH_PROCESS_SUBST(, int ctl))
>  {
>         int pip[2];
>         struct job *jp;
> +#if BASH_PROCESS_SUBST
> +       /* determine end of pipe used by parent (ip) and child (ic) */
> +       const int ip = (ctl == CTLTOPROC);
> +       const int ic = !(ctl == CTLTOPROC);
> +#else
> +       const int ctl = CTLBACKQ;
> +       const int ip = 0;
> +       const int ic = 1;
> +#endif
>
>         result->fd = -1;
>         result->buf = NULL;
> @@ -6552,15 +6624,17 @@ evalbackcmd(union node *n, struct backcmd *result)
>
>         if (pipe(pip) < 0)
>                 ash_msg_and_raise_perror("can't create pipe");
> -       jp = makejob(/*n,*/ 1);
> -       if (forkshell(jp, n, FORK_NOJOB) == 0) {
> +       /* process substitution uses NULL job/node, like openhere() */
> +       jp = (ctl == CTLBACKQ) ? makejob(/*n,*/ 1) : NULL;
> +       if (forkshell(jp, (ctl == CTLBACKQ) ? n : NULL, FORK_NOJOB) == 0) {
>                 /* child */
>                 FORCE_INT_ON;
> -               close(pip[0]);
> -               if (pip[1] != 1) {
> -                       /*close(1);*/
> -                       dup2_or_raise(pip[1], 1);
> -                       close(pip[1]);
> +               close(pip[ip]);
> +               /* ic is index of child end of pipe *and* fd to connect it to */
> +               if (pip[ic] != ic) {
> +                       /*close(ic);*/
> +                       dup2_or_raise(pip[ic], ic);
> +                       close(pip[ic]);
>                 }
>  /* TODO: eflag clearing makes the following not abort:
>   *  ash -c 'set -e; z=$(false;echo foo); echo $z'
> @@ -6576,8 +6650,18 @@ evalbackcmd(union node *n, struct backcmd *result)
>                 /* NOTREACHED */
>         }
>         /* parent */
> -       close(pip[1]);
> -       result->fd = pip[0];
> +#if BASH_PROCESS_SUBST
> +       if (ctl != CTLBACKQ) {
> +               int fd = fcntl(pip[ip], F_DUPFD, 64);
> +               if (fd > 0) {
> +                       close(pip[ip]);
> +                       pip[ip] = fd;
> +               }
> +               pushfd(pip[ip]);
> +       }
> +#endif
> +       close(pip[ic]);
> +       result->fd = pip[ip];
>         result->jp = jp;
>
>   out:
> @@ -6589,8 +6673,11 @@ evalbackcmd(union node *n, struct backcmd *result)
>   * Expand stuff in backwards quotes.
>   */
>  static void
> -expbackq(union node *cmd, int flag)
> +expbackq(union node *cmd, int flag IF_BASH_PROCESS_SUBST(, int ctl))
>  {
> +#if !BASH_PROCESS_SUBST
> +       const int ctl = CTLBACKQ;
> +#endif
>         struct backcmd in;
>         int i;
>         char buf[128];
> @@ -6605,9 +6692,15 @@ expbackq(union node *cmd, int flag)
>         INT_OFF;
>         startloc = expdest - (char *)stackblock();
>         pushstackmark(&smark, startloc);
> -       evalbackcmd(cmd, &in);
> +       evalbackcmd(cmd, &in IF_BASH_PROCESS_SUBST(, ctl));
>         popstackmark(&smark);
>
> +       if (ctl != CTLBACKQ) {
> +               sprintf(buf, DEV_FD_PREFIX"%d", in.fd);
> +               strtodest(buf, BASESYNTAX);
> +               goto done;
> +       }
> +
>         p = in.buf;
>         i = in.nleft;
>         if (i == 0)
> @@ -6629,6 +6722,7 @@ expbackq(union node *cmd, int flag)
>                 close(in.fd);
>                 back_exitstatus = waitforjob(in.jp);
>         }
> + IF_BASH_PROCESS_SUBST(done:)
>         INT_ON;
>
>         /* Eat all trailing newlines */
> @@ -6716,6 +6810,10 @@ argstr(char *p, int flag)
>                 CTLESC,
>                 CTLVAR,
>                 CTLBACKQ,
> +#if BASH_PROCESS_SUBST
> +               CTLTOPROC,
> +               CTLFROMPROC,
> +#endif
>  #if ENABLE_FEATURE_SH_MATH
>                 CTLARI,
>                 CTLENDARI,
> @@ -6815,8 +6913,12 @@ argstr(char *p, int flag)
>                         p = evalvar(p, flag | inquotes);
>                         TRACE(("argstr: evalvar:'%s'\n", (char *)stackblock()));
>                         goto start;
> +#if BASH_PROCESS_SUBST
> +               case CTLTOPROC:
> +               case CTLFROMPROC:
> +#endif
>                 case CTLBACKQ:
> -                       expbackq(argbackq->n, flag | inquotes);
> +                       expbackq(argbackq->n, flag | inquotes IF_BASH_PROCESS_SUBST(, c));
>                         goto start;
>  #if ENABLE_FEATURE_SH_MATH
>                 case CTLARI:
> @@ -12203,8 +12305,9 @@ realeofmark(const char *eofmark)
>  #define CHECKEND()      {goto checkend; checkend_return:;}
>  #define PARSEREDIR()    {goto parseredir; parseredir_return:;}
>  #define PARSESUB()      {goto parsesub; parsesub_return:;}
> -#define PARSEBACKQOLD() {oldstyle = 1; goto parsebackq; parsebackq_oldreturn:;}
> -#define PARSEBACKQNEW() {oldstyle = 0; goto parsebackq; parsebackq_newreturn:;}
> +#define PARSEBACKQOLD() {style = OLD; goto parsebackq; parsebackq_oldreturn:;}
> +#define PARSEBACKQNEW() {style = NEW; goto parsebackq; parsebackq_newreturn:;}
> +#define PARSEPROCSUB()  {style = PSUB; goto parsebackq; parsebackq_psreturn:;}
>  #define PARSEARITH()    {goto parsearith; parsearith_return:;}
>  static int
>  readtoken1(int c, int syntax, char *eofmark, int striptabs)
> @@ -12215,7 +12318,9 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
>         size_t len;
>         struct nodelist *bqlist;
>         smallint quotef;
> -       smallint oldstyle;
> +       smallint style;
> +       enum { OLD, NEW, PSUB };
> +#define oldstyle style == OLD
>         smallint pssyntax;   /* we are expanding a prompt string */
>         IF_BASH_DOLLAR_SQUOTE(smallint bash_dollar_squote = 0;)
>         /* syntax stack */
> @@ -12396,6 +12501,15 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
>                                                 c = 0x100 + '>'; /* flag &> */
>                                         pungetc();
>                                 }
> +#endif
> +#if BASH_PROCESS_SUBST
> +                               if (c == '<' || c == '>') {
> +                                       if (pgetc() == '(') {
> +                                               PARSEPROCSUB();
> +                                               break;
> +                                       }
> +                                       pungetc();
> +                               }
>  #endif
>                                 goto endword;   /* exit outer loop */
>                         }
> @@ -12881,9 +12995,18 @@ parsebackq: {
>                 memcpy(out, str, savelen);
>                 STADJUST(savelen, out);
>         }
> -       USTPUTC(CTLBACKQ, out);
> +#if BASH_PROCESS_SUBST
> +       if (style == PSUB)
> +               USTPUTC(c == '<' ? CTLFROMPROC : CTLTOPROC, out);
> +       else
> +#endif
> +               USTPUTC(CTLBACKQ, out);
>         if (oldstyle)
>                 goto parsebackq_oldreturn;
> +#if BASH_PROCESS_SUBST
> +       else if (style == PSUB)
> +               goto parsebackq_psreturn;
> +#endif
>         goto parsebackq_newreturn;
>  }
>
> @@ -13334,6 +13457,9 @@ cmdloop(int top)
>  #if JOBS
>                 if (doing_jobctl)
>                         showjobs(SHOW_CHANGED|SHOW_STDERR);
> +#endif
> +#if BASH_PROCESS_SUBST
> +               unwindredir(NULL);
>  #endif
>                 inter = 0;
>                 if (iflag && top) {
> diff --git a/shell/ash_test/ash-psubst/bash_procsub.right b/shell/ash_test/ash-psubst/bash_procsub.right
> new file mode 100644
> index 000000000..aa16a96be
> --- /dev/null
> +++ b/shell/ash_test/ash-psubst/bash_procsub.right
> @@ -0,0 +1,9 @@
> +hello 1
> +hello 2
> +hello 3
> +<(echo "hello 0")
> +hello 4
> +HI THERE
> +hello error
> +hello error
> +hello stderr
> diff --git a/shell/ash_test/ash-psubst/bash_procsub.tests b/shell/ash_test/ash-psubst/bash_procsub.tests
> new file mode 100755
> index 000000000..63b836782
> --- /dev/null
> +++ b/shell/ash_test/ash-psubst/bash_procsub.tests
> @@ -0,0 +1,33 @@
> +# simplest case
> +cat <(echo "hello 1")
> +
> +# can have more than one
> +cat <(echo "hello 2") <(echo "hello 3")
> +
> +# doesn't work in quotes
> +echo "<(echo \"hello 0\")"
> +
> +# process substitution can be nested inside command substitution
> +echo $(cat <(echo "hello 4"))
> +
> +# example from http://wiki.bash-hackers.org/syntax/expansion/proc_subst
> +# process substitutions can be passed to a function as parameters or
> +# variables
> +f() {
> +       cat "$1" >"$x"
> +}
> +x=>(tr '[:lower:]' '[:upper:]') f <(echo 'hi there')
> +
> +# process substitution can be combined with redirection on exec
> +rm -f err
> +# save stderr
> +exec 4>&2
> +# copy stderr to a file
> +exec 2> >(tee err)
> +echo "hello error" >&2
> +sync
> +# restore stderr
> +exec 2>&4
> +cat err
> +rm -f err
> +echo "hello stderr" >&2
> --
> 2.26.2
>
> _______________________________________________
> busybox mailing list
> busybox at busybox.net
> http://lists.busybox.net/mailman/listinfo/busybox


More information about the busybox mailing list