fixes for posix-mode alias expansion in command substitutions; fix for multiple function definitions with here-documents inside command substitutions

CWRU/CWRU.chlog

@@ -4144,3 +4144,54 @@ parse.y
 	  reported by feng xiangjun <fengxj325@gmail.com>
 	- parse_string_to_word_list,parse_string_to_command: make sure to set
 	  PST_STRING in parser_flags since we're resetting expand_aliases
+
+				   10/17
+				   -----
+print_cmd.c
+	- print_function_def, named_function_string: if the function doesn't
+	  have any here-documents, unset was_heredoc after printing the
+	  closing brace so we can add a separator if there is another command
+	  following the function definition. Fixes bug with multiple function
+	  definitions and here-documents reported by D630 <d630@posteo.net>
+	- make_command_string_internal: if we're printing a command
+	  substitution, make sure to preserve newlines in the comsub so we
+	  avoid problems with re-parsing function defs or command substitutions
+	  containing multiple compound commands that need to be separated
+	  by newlines, but don't double them up
+
+parse.y
+	- P_ARITH: new parse_matched_pair flags value; used to jump to
+	  parse_comsub if we see a $( inside a $(( )), $[ ], or (( ))
+	- parse_matched_pair,parse_comsub,parse_arith_command,read_token_word:
+	  make sure to call parse_matched_pair with P_ARITH if we're parsing
+	  an arithmetic command or expansion
+	- parse_matched_pair: if we see $( inside a call with P_ARITH in the
+	  flags argument, jump to parse_dollar_word and call parse_comsub to
+	  parse it. The rest of the steps assume that parse_comsub has
+	  expanded aliases (or not) appropriately. Fixes bug 2 from
+	  https://bugzilla.redhat.com/show_bug.cgi?id=2134307
+
+				   10/18
+				   -----
+builtins/evalstring.c
+	- parse_and_execute: we play tricks in the parser and in command_substitute
+	  to turn expand_aliases on and off depending on the comsub parser
+	  pass and whether or not we're in posix mode. Since this should only
+	  matter for parsing, when we're processing a command substitution, we
+	  set expand_aliases to the global flag value after parsing the
+	  command and before executing it, and restore it if the global flag
+	  value isn't changed during execution. Fixes bug 1 from
+	  https://bugzilla.redhat.com/show_bug.cgi?id=2134307
+
+command.h
+	- PF_BACKQUOTE: new flag value, only used by command_substitute to
+	  differentiate between `` and $() forms. The former does not have
+	  aliases expanded by parse_comsub in posix mode, so needs to expand
+	  them in parse_and_execute like default mode
+
+subst.c
+	- command_substitute: don't modify expand_aliases if PF_BACKQUOTE is
+	  included in flags, since aliases haven't been expanded in the
+	  command string (it was run through parse.y:parse_matched_pair())
+	- expand_word_internal: pass PF_BACKQUOTE to command_substitute() if
+	  expanding a `` command substitution

MANIFEST

@@ -1003,6 +1003,7 @@ tests/comsub3.sub	f
 tests/comsub4.sub	f
 tests/comsub5.sub	f
 tests/comsub6.sub	f
+tests/comsub7.sub	f
 tests/comsub-eof.tests	f
 tests/comsub-eof0.sub	f
 tests/comsub-eof1.sub	f
@@ -1176,6 +1177,7 @@ tests/heredoc4.sub	f
 tests/heredoc5.sub	f
 tests/heredoc6.sub	f
 tests/heredoc7.sub	f
+tests/heredoc8.sub	f
 tests/herestr.tests	f
 tests/herestr.right	f
 tests/herestr1.sub	f

builtins/evalstring.c

@@ -431,6 +431,8 @@ parse_and_execute (string, from_file, flags)
 
       if (parse_command () == 0)
 	{
+	  int local_expalias, local_alflag;
+
 	  if ((flags & SEVAL_PARSEONLY) || (interactive_shell == 0 && read_but_dont_execute))
 	    {
 	      last_result = EXECUTION_SUCCESS;
@@ -507,6 +509,19 @@ parse_and_execute (string, from_file, flags)
 		}
 #endif /* ONESHOT */
 
+	      /* We play tricks in the parser and command_substitute() turning
+		 expand_aliases on and off depending on which parsing pass and
+		 whether or not we're in posix mode. This only matters for
+		 parsing, and we let the higher layers deal with that. We just
+		 want to ensure that expand_aliases is set to the appropriate
+		 global value when we go to execute this command, so we save
+		 and restore it around the execution (we don't restore it if
+		 the global value of the flag (expaliases_flag) changes). */
+	      local_expalias = expand_aliases;
+	      local_alflag = expaliases_flag;
+	      if (subshell_environment & SUBSHELL_COMSUB)
+		expand_aliases = expaliases_flag;
+
 	      /* See if this is a candidate for $( <file ). */
 	      if (startup_state == 2 &&
 		  (subshell_environment & SUBSHELL_COMSUB) &&
@@ -524,6 +539,10 @@ parse_and_execute (string, from_file, flags)
 	      dispose_fd_bitmap (bitmap);
 	      discard_unwind_frame ("pe_dispose");
 
+	      /* If the global value didn't change, we restore what we had. */
+	      if ((subshell_environment & SUBSHELL_COMSUB) && local_alflag == expaliases_flag)
+		expand_aliases = local_expalias;
+
 	      if (flags & SEVAL_ONECMD)
 		{
 		  reset_parser ();

command.h

@@ -114,6 +114,7 @@ enum command_type { cm_for, cm_case, cm_while, cm_if, cm_simple, cm_select,
 #define PF_COMPLETE	0x10	/* same as W_COMPLETE, sets SX_COMPLETE */
 #define PF_EXPANDRHS	0x20	/* same as W_EXPANDRHS */
 #define PF_ALLINDS	0x40	/* array, act as if [@] was supplied */
+#define PF_BACKQUOTE	0x80	/* differentiate `` from $() for command_substitute */
 
 /* Possible values for subshell_environment */
 #define SUBSHELL_ASYNC	0x01	/* subshell caused by `command &' */

parse.y

@@ -3613,6 +3613,7 @@ tokword:
 #define P_BACKQUOTE	0x0010	/* parsing a backquoted command substitution */
 #define P_ARRAYSUB	0x0020	/* parsing a [...] array subscript for assignment */
 #define P_DOLBRACE	0x0040	/* parsing a ${...} construct */
+#define P_ARITH		0x0080	/* parsing a $(( )) arithmetic expansion */
 
 /* Lexical state while parsing a grouping construct or $(...). */
 #define LEX_WASDOL	0x0001
@@ -3911,6 +3912,9 @@ parse_matched_pair (qc, open, close, lenp, flags)
 	    }
 	  else if ((flags & (P_ARRAYSUB|P_DOLBRACE)) && (tflags & LEX_WASDOL) && (ch == '(' || ch == '{' || ch == '['))	/* ) } ] */
 	    goto parse_dollar_word;
+	  else if ((flags & P_ARITH) && (tflags & LEX_WASDOL) && ch == '(') /*)*/
+	    /* $() inside $(( ))/$[ ] */
+	    goto parse_dollar_word;
 #if defined (PROCESS_SUBSTITUTION)
 	  /* XXX - technically this should only be recognized at the start of
 	     a word */
@@ -3941,7 +3945,7 @@ parse_dollar_word:
 	  else if (ch == '{')		/* } */
 	    nestret = parse_matched_pair (0, '{', '}', &nestlen, P_FIRSTCLOSE|P_DOLBRACE|rflags);
 	  else if (ch == '[')		/* ] */
-	    nestret = parse_matched_pair (0, '[', ']', &nestlen, rflags);
+	    nestret = parse_matched_pair (0, '[', ']', &nestlen, rflags|P_ARITH);
 
 	  CHECK_NESTRET_ERROR ();
 	  APPEND_NESTRET ();
@@ -4080,7 +4084,7 @@ parse_comsub (qc, open, close, lenp, flags)
       peekc = shell_getc (1);
       shell_ungetc (peekc);
       if (peekc == '(')		/*)*/
-	return (parse_matched_pair (qc, open, close, lenp, 0));
+	return (parse_matched_pair (qc, open, close, lenp, P_ARITH));
     }
 
 /*itrace("parse_comsub: qc = `%c' open = %c close = %c", qc, open, close);*/
@@ -4504,7 +4508,7 @@ parse_arith_cmd (ep, adddq)
   int ttoklen;
 
   exp_lineno = line_number;
-  ttok = parse_matched_pair (0, '(', ')', &ttoklen, 0);
+  ttok = parse_matched_pair (0, '(', ')', &ttoklen, P_ARITH);
   rval = 1;
   if (ttok == &matched_pair_error)
     return -1;
@@ -5022,7 +5026,7 @@ read_token_word (character)
 		  pop_delimiter (dstack);
 		}
 	      else
-		ttok = parse_matched_pair (cd, '[', ']', &ttoklen, 0);
+		ttok = parse_matched_pair (cd, '[', ']', &ttoklen, P_ARITH);
 	      if (ttok == &matched_pair_error)
 		return -1;		/* Bail immediately. */
 	      RESIZE_MALLOCED_BUFFER (token, token_index, ttoklen + 3,

po/LINGUAS

@@ -1,2 +1,2 @@
 # Set of available languages.
-en@quot en@boldquot af bg ca cs da de el eo es et fi fr ga gl hr hu id it ja ko lt nb nl pl pt pt_BR ro ru sk sl sr sv tr uk vi zh_CN zh_TW
+en@quot en@boldquot af bg ca cs da de el eo es et fi fr ga gl hr hu id it ja ka ko lt nb nl pl pt pt_BR ro ru sk sl sr sv tr uk vi zh_CN zh_TW

print_cmd.c

@@ -297,10 +297,12 @@ make_command_string_internal (command)
 	    case '\n':				/* special case this */
 	      {
 		char c = command->value.Connection->connector;
+		int was_newline;
 
 		s[0] = printing_comsub ? c : ';';
 		s[1] = '\0';
 
+		was_newline = deferred_heredocs == 0 && was_heredoc == 0 && c == '\n';
 		if (deferred_heredocs == 0)
 		  {
 		    if (was_heredoc == 0)
@@ -314,6 +316,8 @@ make_command_string_internal (command)
 
 		if (inside_function_def)
 		  cprintf ("\n");
+		else if (printing_comsub && c == '\n' && was_newline == 0)
+		  cprintf ("\n");	/* preserve newlines in comsubs but don't double them */
 		else
 		  {
 		    if (c == ';')
@@ -1339,7 +1343,7 @@ print_function_def (func)
   add_unwind_protect (reset_locals, 0);
 
   indent (indentation);
-  cprintf ("{ \n");
+  cprintf ("{ \n");	/* } */
 
   inside_function_def++;
   indentation += indentation_amount;
@@ -1366,7 +1370,11 @@ print_function_def (func)
       cmdcopy->redirects = func_redirects;
     }
   else
-    newline ("}");
+    {
+      /* { */
+      newline ("}");
+      was_heredoc = 0;		/* not printing any here-documents now */
+    }
 
   dispose_command (cmdcopy);
 }
@@ -1416,7 +1424,7 @@ named_function_string (name, command, flags)
 
   inside_function_def++;
 
-  cprintf ((flags & FUNC_MULTILINE) ? "{ \n" : "{ ");
+  cprintf ((flags & FUNC_MULTILINE) ? "{ \n" : "{ ");	/* }} */
 
   cmdcopy = copy_command (command);
   /* Take any redirections specified in the function definition (which should
@@ -1443,7 +1451,10 @@ named_function_string (name, command, flags)
       cmdcopy->redirects = func_redirects;
     }
   else
-    newline ("}");
+    {	/* { */
+      newline ("}");
+      was_heredoc = 0;
+    }
 
   result = the_printed_command;
 

subst.c

@@ -7128,8 +7128,12 @@ command_substitute (string, quoted, flags)
       remove_quoted_escapes (string);
 
       /* We want to expand aliases on this pass if we are not in posix mode
-	 for backwards compatibility. */
-      if (expand_aliases)
+	 for backwards compatibility. parse_and_execute() takes care of
+	 setting expand_aliases back to the global value when executing the
+	 parsed string. We only do this for $(...) command substitution,
+	 since that is what parse_comsub handles; `` comsubs are processed
+	 using parse.y:parse_matched_pair(). */
+      if (expand_aliases && (flags & PF_BACKQUOTE) == 0)
         expand_aliases = posixly_correct == 0;
 
       startup_state = 2;	/* see if we can avoid a fork */
@@ -11297,7 +11301,7 @@ add_string:
 	    else
 	      {
 		de_backslash (temp);
-		tword = command_substitute (temp, quoted, 0);
+		tword = command_substitute (temp, quoted, PF_BACKQUOTE);
 		temp1 = tword ? tword->word : (char *)NULL;
 		if (tword)
 		  dispose_word_desc (tword);

tests/comsub.right

@@ -77,3 +77,16 @@ Mon Aug 29 20:03:02 EDT 2022
 hey after x
 ./comsub6.sub: line 40: syntax error near unexpected token `)'
 ./comsub6.sub: line 40: `math1)'
+123
+123
+0
+123
+123
+0
+Mon Aug 29 20:03:02 EDT 2022
+Mon Aug 29 20:03:02 EDT 2022
+Mon Aug 29 20:03:02 EDT 2022
+Mon Aug 29 20:03:02 EDT 2022
+123
+before 123
+in for 123

tests/comsub.tests

@@ -82,3 +82,4 @@ ${THIS_SH} ./comsub3.sub
 ${THIS_SH} ./comsub4.sub
 ${THIS_SH} ./comsub5.sub
 ${THIS_SH} ./comsub6.sub
+${THIS_SH} ./comsub7.sub

tests/comsub7.sub

@@ -0,0 +1,64 @@
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# posix-mode alias expansion in command substitutions within other constructs
+
+DATE='Mon Aug 29 20:03:02 EDT 2022'
+shopt -s expand_aliases
+
+alias number="echo 123"
+
+echo $(number)
+echo $(( $(number) ))
+(( $(number) )) ; echo $?
+
+set -o posix
+echo $(number)
+echo $(( $(number) ))
+(( $(number) )) ; echo $?
+set +o posix
+
+# have to turn it back on after leaving posix mode
+shopt -s expand_aliases
+
+alias my_alias='echo $DATE'
+
+echo $(eval my_alias)
+echo $(my_alias)
+
+set -o posix
+echo $(eval my_alias)
+echo $(my_alias)
+set +o posix ; shopt -s expand_aliases
+
+alias e=echo
+alias v='e 123'
+
+set -o posix
+echo $(v)
+echo $(echo before ; v)
+echo $(for f in 0; do
+echo in for
+done; v)
+set +o posix ; shopt -s expand_aliases
+
+alias let='let --'
+
+let '1 == 1'
+: $(let '1 == 1')
+
+set -o posix
+let '1 == 1'
+: $(let '1 == 1')
+set +o posix ; shopt -s expand_aliases

tests/heredoc.tests

@@ -142,7 +142,7 @@ ${THIS_SH} ./heredoc6.sub
 
 # interaction between here-documents and command substitutions
 ${THIS_SH} ./heredoc7.sub
-
+${THIS_SH} ./heredoc8.sub
 
 echo $(
 	cat <<< "comsub here-string"

tests/heredoc8.sub

@@ -0,0 +1,62 @@
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# problems with comsub printing and re-parsing with here-documents and compound
+# commands in bash-5.2
+
+z=$(
+    f1() {
+        c <<-!
+	!
+    }
+    f2() {
+        :
+    }
+)
+
+z=$(
+    f1() {
+        c
+    }
+    f2() {
+        :
+    }
+)
+
+z=$(
+    f1() {
+        c <<-!
+	!
+    }
+	:
+)
+
+z=$(
+    {
+        : <<-!
+	!
+    }
+    {
+        :
+    }
+)
+
+z=$(
+    for f in 0; do
+        : <<-!
+	!
+    done
+    {
+        :
+    }
+)