fix up parser flags command substitution parsing inherits; using temp files for here-documents is now a compatibility mode option

CWRU/CWRU.chlog

@@ -3571,3 +3571,23 @@ redir.c
 	  earlier, use tempfiles for all here-documents and here-strings. From
 	  a bug-bash discussion started by Sam Liddicott <sam@liddicott.com>
 
+				   4/26
+				   ----
+parse.y
+	- parse_comsub: non-interactive shells exit on a syntax error while
+	  parsing the command substitution
+	- parse_comsub: unset additional PARSER_STATE flags before calling
+	  yyparse(). Inspired by https://bugs.gentoo.org/837203; unsetting
+	  PST_COMPASSIGN is the fix for that bug
+	- parse_string_to_word_list: use save_parser_state/restore_parser_state
+	  instead of saving pieces of the shell state in individual variables
+	- parse_compound_assignment: use save_parser_state/restore_parser_state
+	  instead of saving pieces of the shell state in individual variables
+	- parse_compound_assignment: unset additional PARSER_STATE flags before
+	  calling read_token(); set esacs_needed_count and expecting_in_token
+	  to 0 like in parse_comsub() since read_token can use them
+
+				   4/27
+				   ----
+lib/sh/strvis.c
+	- strivs: changes to handle being compiled without multibyte support

MANIFEST

@@ -1013,6 +1013,7 @@ tests/comsub-posix1.sub	f
 tests/comsub-posix2.sub	f
 tests/comsub-posix3.sub	f
 tests/comsub-posix5.sub	f
+tests/comsub-posix6.sub	f
 tests/cond.tests	f
 tests/cond.right	f
 tests/cond-regexp1.sub	f

lib/glob/glob.c

@@ -121,7 +121,7 @@ void wcdequote_pathname PARAMS((wchar_t *));
 static void wdequote_pathname PARAMS((char *));
 static void dequote_pathname PARAMS((char *));
 #else
-#  define dequote_pathname udequote_pathname
+#  define dequote_pathname(p) udequote_pathname(p)
 #endif
 static int glob_testdir PARAMS((char *, int));
 static char **glob_dir_to_array PARAMS((char *, char **, int));

lib/sh/strvis.c

@@ -69,7 +69,11 @@ sh_charvis (s, sindp, slen, ret, rindp)
   ri = *rindp;
   c = s[*sindp];
 
+#if defined (HANDLE_MULTIBYTE)
   send = (locale_mb_cur_max > 1) ? s + slen : 0;
+#else
+  send = 0;
+#endif
 
   if (SAFECHAR (c))
     {
@@ -88,10 +92,12 @@ sh_charvis (s, sindp, slen, ret, rindp)
       ret[ri++] = UNCTRL (c);
       si++;
     }
+#if defined (HANDLE_MULTIBYTE)
   else if (locale_utf8locale && (c & 0x80))
     COPY_CHAR_I (ret, ri, s, send, si);
   else if (locale_mb_cur_max > 1 && is_basic (c) == 0)
     COPY_CHAR_I (ret, ri, s, send, si);
+#endif
   else if (META_CHAR (c))
     {
       ret[ri++] = 'M';

parse.y

@@ -4063,7 +4063,12 @@ parse_comsub (qc, open, close, lenp, flags)
   pushed_string_list = (STRING_SAVER *)NULL;
 
   /* State flags we don't want to persist into command substitutions. */
-  parser_state &= ~(PST_REGEXP|PST_EXTPAT|PST_CONDCMD|PST_CONDEXPR);
+  parser_state &= ~(PST_REGEXP|PST_EXTPAT|PST_CONDCMD|PST_CONDEXPR|PST_COMPASSIGN);
+  /* Could do PST_CASESTMT too, but that also affects history. Setting
+     expecting_in_token below should take care of the parsing requirements.
+     Unsetting PST_REDIRLIST isn't strictly necessary because of how we set
+     token_to_read below, but we do it anyway. */
+  parser_state &= ~(PST_CASEPAT|PST_ALEXPNEXT|PST_SUBSHELL|PST_REDIRLIST);
   /* State flags we want to set for this run through the parser. */
   parser_state |= PST_CMDSUBST|PST_EOFTOKEN|PST_NOEXPAND;
 
@@ -4102,7 +4107,14 @@ parse_comsub (qc, open, close, lenp, flags)
   else if (r != 0)
     {
       /* parser_error (start_lineno, _("could not parse command substitution")); */
-      jump_to_top_level (DISCARD);      
+      /* Non-interactive shells exit on parse error in a command substitution. */
+      if (last_command_exit_value == 0)
+	last_command_exit_value = EXECUTION_FAILURE;
+      set_exit_status (last_command_exit_value);
+      if (interactive_shell == 0)
+	jump_to_top_level (FORCE_EOF);	/* This is like reader_loop() */
+      else
+	jump_to_top_level (DISCARD);
     }
 
   if (current_token != shell_eof_token)
@@ -6285,31 +6297,26 @@ parse_string_to_word_list (s, flags, whom)
      const char *whom;
 {
   WORD_LIST *wl;
-  int tok, orig_current_token, orig_line_number, orig_input_terminator;
-  int orig_line_count, orig_parser_state;
-  int old_echo_input, old_expand_aliases, ea;
-#if defined (HISTORY)
-  int old_remember_on_history, old_history_expansion_inhibited;
-#endif
+  int tok, orig_current_token, orig_line_number;
+  int orig_parser_state;
+  sh_parser_state_t ps;
+  int ea;
 
 #if defined (HISTORY)
-  old_remember_on_history = remember_on_history;
-#  if defined (BANG_HISTORY)
-  old_history_expansion_inhibited = history_expansion_inhibited;
-#  endif
   bash_history_disable ();
 #endif
 
   orig_line_number = line_number;
-  orig_line_count = current_command_line_count;
-  orig_input_terminator = shell_input_line_terminator;
-  old_echo_input = echo_input_at_read;
-  old_expand_aliases = expand_aliases;
+  save_parser_state (&ps);
 
   push_stream (1);
   if (ea = expanding_alias ())
     parser_save_alias ();
-  last_read_token = WORD;		/* WORD to allow reserved words here */
+
+  /* WORD to avoid parsing reserved words as themselves and just parse them as
+     WORDs. */
+  last_read_token = WORD;
+
   current_command_line_count = 0;
   echo_input_at_read = expand_aliases = 0;
 
@@ -6318,9 +6325,11 @@ parse_string_to_word_list (s, flags, whom)
 
   if (flags & 1)
     {
-      orig_parser_state = parser_state;
-      parser_state |= PST_COMPASSIGN|PST_REPARSE;
+      orig_parser_state = parser_state;		/* XXX - not needed? */
+      /* State flags we don't want to persist into compound assignments. */
       parser_state &= ~PST_NOEXPAND;	/* parse_comsub sentinel */
+      /* State flags we want to set for this run through the tokenizer. */
+      parser_state |= PST_COMPASSIGN|PST_REPARSE;
     }
 
   while ((tok = read_token (READ)) != yacc_EOF)
@@ -6350,21 +6359,10 @@ parse_string_to_word_list (s, flags, whom)
   if (ea)
     parser_restore_alias ();
 
-#if defined (HISTORY)
-  remember_on_history = old_remember_on_history;
-#  if defined (BANG_HISTORY)
-  history_expansion_inhibited = old_history_expansion_inhibited;
-#  endif /* BANG_HISTORY */
-#endif /* HISTORY */
-
-  echo_input_at_read = old_echo_input;
-  expand_aliases = old_expand_aliases;
-
-  current_command_line_count = orig_line_count;
-  shell_input_line_terminator = orig_input_terminator;
+  restore_parser_state (&ps);
 
   if (flags & 1)
-    parser_state = orig_parser_state;
+    parser_state = orig_parser_state;	/* XXX - not needed? */
 
   if (wl == &parse_string_error)
     {
@@ -6383,29 +6381,31 @@ parse_compound_assignment (retlenp)
      int *retlenp;
 {
   WORD_LIST *wl, *rl;
-  int tok, orig_line_number, orig_last_token, assignok;
-  size_t orig_token_size;
-  int orig_parser_state;
-  char *saved_token, *ret;
+  int tok, orig_line_number, assignok;
+  sh_parser_state_t ps;
+  char *ret;
 
-  saved_token = token;
-  orig_token_size = token_buffer_size;
   orig_line_number = line_number;
-  orig_last_token = last_read_token;
-  orig_parser_state = parser_state;
+  save_parser_state (&ps);
 
-  last_read_token = WORD;	/* WORD to allow reserved words here */
+  /* WORD to avoid parsing reserved words as themselves and just parse them as
+     WORDs. Plus it means we won't be in a command position and so alias
+     expansion won't happen. */
+  last_read_token = WORD;
 
   token = (char *)NULL;
   token_buffer_size = 0;
+  wl = (WORD_LIST *)NULL;	/* ( */
 
   assignok = parser_state&PST_ASSIGNOK;		/* XXX */
 
-  wl = (WORD_LIST *)NULL;	/* ( */
-  orig_parser_state = parser_state;
-  parser_state &= ~PST_NOEXPAND;
+  /* State flags we don't want to persist into compound assignments. */
+  parser_state &= ~(PST_NOEXPAND|PST_CONDCMD|PST_CONDEXPR|PST_REGEXP|PST_EXTPAT);
+  /* State flags we want to set for this run through the tokenizer. */
   parser_state |= PST_COMPASSIGN;
 
+  esacs_needed_count = expecting_in_token = 0;
+
   while ((tok = read_token (READ)) != ')')
     {
       if (tok == '\n')			/* Allow newlines in compound assignments */
@@ -6429,11 +6429,7 @@ parse_compound_assignment (retlenp)
       wl = make_word_list (yylval.word, wl);
     }
 
-  FREE (token);
-  token = saved_token;
-  token_buffer_size = orig_token_size;
-
-  parser_state = orig_parser_state;
+  restore_parser_state (&ps);
 
   if (wl == &parse_string_error)
     {
@@ -6445,8 +6441,6 @@ parse_compound_assignment (retlenp)
 	jump_to_top_level (DISCARD);
     }
 
-  last_read_token = orig_last_token;		/* XXX - was WORD? */
-
   if (wl)
     {
       rl = REVERSE_LIST (wl, WORD_LIST *);

tests/comsub-posix.right

@@ -61,7 +61,6 @@ here-doc terminated with a parenthesis
 line terminated with a backslash
 ./comsub-posix1.sub: line 1: syntax error near unexpected token `)'
 ./comsub-posix1.sub: line 1: `echo $( if x; then echo foo )'
-after
 swap32_posix is a function
 swap32_posix () 
 { 
@@ -77,16 +76,23 @@ swap32_posix ()
                 ));
     done
 }
-./comsub-posix5.sub: line 37: syntax error near unexpected token `done'
-./comsub-posix5.sub: line 37: `: $(case x in x) ;; x) done esac)'
-./comsub-posix5.sub: line 38: syntax error near unexpected token `done'
-./comsub-posix5.sub: line 38: `: $(case x in x) ;; x) done ;; esac)'
-./comsub-posix5.sub: line 39: syntax error near unexpected token `esac'
-./comsub-posix5.sub: line 39: `: $(case x in x) (esac) esac)'
+bash: -c: line 1: syntax error near unexpected token `done'
+bash: -c: line 1: `: $(case x in x) ;; x) done esac)'
+bash: -c: line 1: syntax error near unexpected token `done'
+bash: -c: line 1: `: $(case x in x) ;; x) done ;; esac)'
+bash: -c: line 1: syntax error near unexpected token `esac'
+bash: -c: line 1: `: $(case x in x) (esac) esac)'
 bash: -c: line 1: syntax error near unexpected token `in'
 bash: -c: line 1: `: $(case x in esac|in) foo;; esac)'
 bash: -c: line 1: syntax error near unexpected token `done'
 bash: -c: line 1: `: $(case x in x) ;; x) done)'
+case: -c: line 3: syntax error near unexpected token `esac'
+case: -c: line 3: `$( esac ; bar=foo ; echo "$bar")) echo bad 2;;'
+ok 2
+inside outside
+ok 3
+syntax-error: -c: line 2: syntax error near unexpected token `done'
+syntax-error: -c: line 2: `: $(case x in x) ;; x) done ;; esac)'
 yes
 
 

tests/comsub-posix.tests

@@ -234,13 +234,12 @@ echo $(
 )
 
 ${THIS_SH} ./comsub-posix1.sub
-
 ${THIS_SH} ./comsub-posix2.sub
-
 ${THIS_SH} ./comsub-posix3.sub
 
 #${THIS_SH} ./comsub-posix4.sub
 ${THIS_SH} ./comsub-posix5.sub
+${THIS_SH} ./comsub-posix6.sub
 
 # produced a parse error through bash-4.0-beta2
 : $(echo foo)"

tests/comsub-posix1.sub

@@ -1,3 +1,3 @@
 echo $( if x; then echo foo )
 
-echo after
+echo should not see this

tests/comsub-posix5.sub

@@ -34,9 +34,9 @@
 : $(case x in x) (echo esac) esac)
 
 # these errors should be caught sooner
-: $(case x in x) ;; x) done esac)
-: $(case x in x) ;; x) done ;; esac)
-: $(case x in x) (esac) esac)
+${THIS_SH} -c ': $(case x in x) ;; x) done esac)' bash
+${THIS_SH} -c ': $(case x in x) ;; x) done ;; esac)' bash
+${THIS_SH} -c ': $(case x in x) (esac) esac)' bash
 
 # these are not errors
 : $(case x in x) ;; x) eval done ;; esac)

tests/comsub-posix6.sub

@@ -0,0 +1,43 @@
+: ${THIS_SH:=./bash}
+
+# comsub should not inherit PST_COMPASSIGN
+
+C=($(echo "${A[@]}" | \
+            (while read -d ' ' i; do
+                C=(${C/ ${i%% *} / })
+            done
+            echo ${C[@]})))
+
+# comsub should not inherit PST_CASEPAT
+
+${THIS_SH} -c '
+case foo in
+$( esac ; bar=foo ; echo "$bar")) echo bad 2;;
+*)	echo ok 2;;
+esac
+
+echo we should not see this' case
+
+# comsub should not inherit PST_SUBSHELL
+
+${THIS_SH} -c '( case foo in
+  ( $(echo foo | cat )) echo ok 2;;
+  *) echo bad 2;;
+  esac
+
+  echo $( echo inside ) outside )' subshell
+
+# comsub should not inherit PST_REDIRLIST
+
+${THIS_SH} -c '
+{fd}</dev/null {fd2}<$(foo=/dev/null ; echo $foo) exec
+case $fd2 in
+[0-9]*)	echo ok 3 ;;
+*)	echo bad 3 ;;
+esac' redirlist
+
+# comsub should exit on syntax error while parsing
+${THIS_SH} -c '
+: $(case x in x) ;; x) done ;; esac)
+
+echo after syntax error' syntax-error

tests/errors.right

@@ -54,10 +54,10 @@ umask: usage: umask [-p] [-S] [mode]
 ./errors.tests: line 177: declare: VAR: readonly variable
 ./errors.tests: line 179: declare: unset: not found
 ./errors.tests: line 182: VAR: readonly variable
-./errors.tests: line 185: syntax error near unexpected token `)'
-./errors.tests: line 185: `: $( for z in 1 2 3; do )'
-./errors.tests: line 186: syntax error near unexpected token `done'
-./errors.tests: line 186: `: $( for z in 1 2 3; done )'
+comsub: -c: line 1: syntax error near unexpected token `)'
+comsub: -c: line 1: `: $( for z in 1 2 3; do )'
+comsub: -c: line 1: syntax error near unexpected token `done'
+comsub: -c: line 1: `: $( for z in 1 2 3; done )'
 ./errors.tests: line 189: cd: HOME not set
 ./errors.tests: line 190: cd: /tmp/xyz.bash: No such file or directory
 ./errors.tests: line 192: cd: OLDPWD not set

tests/errors.tests

@@ -181,9 +181,9 @@ declare -p unset
 # iteration variable in a for statement being readonly
 for VAR in 1 2 3 ; do echo $VAR; done
 
-# parser errors
-: $( for z in 1 2 3; do )
-: $( for z in 1 2 3; done )
+# parser errors; caught early so we have to run them in subshells
+${THIS_SH} -c ': $( for z in 1 2 3; do )' comsub
+${THIS_SH} -c ': $( for z in 1 2 3; done )' comsub
 
 # various `cd' errors
 ( unset HOME ; cd )