command substitution parsing now calls the parser recursively

CWRU/CWRU.chlog

@@ -10559,3 +10559,47 @@ builtins/common.[ch]
 
 builtins/set.def
 	- unset_builtin: call builtin_arrayref_flags to set vflags
+
+				   6/19
+				   ----
+
+parse.y
+	- parse_comsub: rewrite to recursively call the parser (yyparse()) and
+	  return a string constructed from the resulting parse tree. Probably
+	  will only work with bison. Error reporting is more accurate about
+	  line numbers and invalid tokens, and command substitution errors 
+	  are caught earlier, before expansion
+	- DOLPAREN: new token, never created by yylex; only ever set by
+	  parse_comsub and xparse_dolparen to indicate we're recursively
+	  calling the parser for a command substitution
+	- comsub: new grammar production that's triggered by DOLPAREN and
+	  parses a command substitution, returning a <command>. It's one of
+	  the possible end states for the top-level parser
+	- grammar: only call rewind_input_string if the shell's input is
+	  coming from a string
+	- shell_ungets: push a string back onto the shell input; only used by
+	  make_here_document for backwards compatibility -- allowing a here-
+	  document to be terminated by a token at the end of a command
+	  substitution
+	- yylex: don't need any more special handling when returning
+	  shell_eof_token, but we keep the clause for future work
+	- read_token_word: don't handle backslashes (leave them in the input
+	  stream) if we are reading a command substitution (PST_NOEXPAND)
+	- reserved_word_acceptable: allow reserved words after DOLPAREN
+	- report_syntax_error: better error handling if we hit EOF while
+	  looking for the ending right paren in a command substitution
+	- parse_string_to_word_list,parse_compound_assignment: make sure to
+	  turn off parse_comsub sentinel temporarily
+
+make_cmd.c
+	- make_here_document: backwards compatibility: if we end a here
+	  document on the same line as the end of a command substitution,
+	  allow the token to terminate the here document (without requiring
+	  a newline) and push the remainder of the line back for the parser
+	  to consume and terminate the command substitution
+
+builtins/evalstring.c
+	- parse_string: if we read shell_eof_token and use it to terminate a
+	  command, rewind the input string here before returning, instead of
+	  guessing where to rewind it in the caller
+

MANIFEST

@@ -1259,6 +1259,7 @@ tests/nquote.right	f
 tests/nquote1.sub	f
 tests/nquote2.sub	f
 tests/nquote3.sub	f
+tests/nquote4.sub	f
 tests/nquote1.tests	f
 tests/nquote1.right	f
 tests/nquote2.tests	f

builtins/evalstring.c

@@ -635,7 +635,11 @@ itrace("parse_string: longjmp executed: code = %d", code);
 	}
 
       if (current_token == yacc_EOF || current_token == shell_eof_token)
+	{
+	  if (current_token == shell_eof_token)
+	    rewind_input_string ();
 	  break;
+	}
     }
 
 out:

doc/bash.1

@@ -3876,8 +3876,6 @@ If the \fBextglob\fP shell option is enabled using the \fBshopt\fP
 builtin, the shell recognizes several extended pattern matching operators.
 In the following description, a \fIpattern-list\fP is a list of one
 or more patterns separated by a \fB|\fP.
-When matching filenames, the \fBdotglob\fP shell option determines
-the set of filenames that are tested, as described above.
 Composite patterns may be formed using one or more of the following
 sub-patterns:
 .sp 1
@@ -3901,6 +3899,16 @@ Matches anything except one of the given patterns
 .RE
 .PD
 .PP
+When matching filenames, the \fBdotglob\fP shell option determines
+the set of filenames that are tested:
+when \fBdotglob\fP is enabled, the set of filenames includes all files
+beginning with ``.'', but ``.'' and ``..'' must be matched by a
+pattern or sub-pattern that begins with a dot;
+when it is disabled, the set does not
+include any filenames beginning with ``.'' unless the pattern
+or sub-pattern begins with a ``.''.
+As above, ``.'' only has a special meaning when matching filenames.
+.PP
 Complicated extended pattern matching against long strings is slow,
 especially when the patterns contain alternations and the strings
 contain multiple matches.

doc/bashref.texi

@@ -2792,6 +2792,17 @@ Matches one of the given patterns.
 Matches anything except one of the given patterns.
 @end table
 
+When matching filenames, the @code{dotglob} shell option determines
+the set of filenames that are tested:
+when @code{dotglob} is enabled, the set of filenames includes all files
+beginning with @samp{.}, but the filenames
+@samp{.} and @samp{..} must be matched by a
+pattern or sub-pattern that begins with a dot;
+when it is disabled, the set does not
+include any filenames beginning with ``.'' unless the pattern
+or sub-pattern begins with a @samp{.}.
+As above, @samp{.} only has a special meaning when matching filenames. 
+
 Complicated extended pattern matching against long strings is slow,
 especially when the patterns contain alternations and the strings
 contain multiple matches.

make_cmd.c

@@ -640,6 +640,15 @@ make_here_document (temp, lineno)
       if (STREQN (line, redir_word, redir_len) && line[redir_len] == '\n')
 	break;
 
+      /* Backwards compatibility here */
+      if (STREQN (line, redir_word, redir_len) && (parser_state & PST_EOFTOKEN) && shell_eof_token && strchr (line+redir_len, shell_eof_token))
+	{
+	  shell_ungets (line + redir_len);
+	  free (full_line);
+	  full_line = 0;
+	  break;
+	}
+
       len = strlen (line);
       if (len + document_index >= document_size)
 	{

tests/comsub-eof.right

@@ -1,17 +1,18 @@
-./comsub-eof0.sub: line 3: unexpected EOF while looking for matching `)'
-./comsub-eof0.sub: line 7: syntax error: unexpected end of file
+./comsub-eof0.sub: line 5: warning: here-document at line 3 delimited by end-of-file (wanted `EOF')
+hi
 hi
 ./comsub-eof2.sub: line 2: warning: here-document at line 1 delimited by end-of-file (wanted `EOF')
 hi
-./comsub-eof3.sub: line 1: unexpected EOF while looking for matching `)'
-./comsub-eof3.sub: line 5: syntax error: unexpected end of file
-./comsub-eof4.sub: line 6: warning: here-document at line 4 delimited by end-of-file (wanted `EOF')
+./comsub-eof3.sub: line 4: warning: here-document at line 1 delimited by end-of-file (wanted `EOF')
+./comsub-eof3.sub: line 5: unexpected EOF while looking for matching `)'
+./comsub-eof3.sub: line 5: unexpected EOF while looking for matching `)'
+./comsub-eof4.sub: line 3: warning: here-document at line 1 delimited by end-of-file (wanted `EOF')
 contents
-./comsub-eof5.sub: line 8: warning: here-document at line 6 delimited by end-of-file (wanted `)')
+./comsub-eof5.sub: line 4: warning: here-document at line 2 delimited by end-of-file (wanted `)')
 hi
-./comsub-eof5.sub: line 13: warning: here-document at line 11 delimited by end-of-file (wanted `EOF')
+./comsub-eof5.sub: line 9: warning: here-document at line 7 delimited by end-of-file (wanted `EOF')
 hi
-./comsub-eof5.sub: line 19: warning: here-document at line 17 delimited by end-of-file (wanted `)')
-./comsub-eof5.sub: line 15: unexpected EOF while looking for matching `)'
-./comsub-eof6.sub: line 1: unexpected EOF while looking for matching `)'
+./comsub-eof5.sub: line 15: warning: here-document at line 13 delimited by end-of-file (wanted `)')
+hi
+./comsub-eof6.sub: command substitution: line 3: unexpected EOF while looking for matching `)'
 

tests/comsub-posix.right

@@ -59,8 +59,8 @@ here-doc with \()
 here-doc terminated with a parenthesis
 ' # or a single back- or doublequote
 line terminated with a backslash
-./comsub-posix1.sub: command substitution: line 2: syntax error near unexpected token `)'
-./comsub-posix1.sub: command substitution: line 2: ` if x; then echo foo )'
+./comsub-posix1.sub: line 1: syntax error near unexpected token `)'
+./comsub-posix1.sub: line 1: `echo $( if x; then echo foo )'
 after
 swap32_posix is a function
 swap32_posix () 
@@ -77,15 +77,18 @@ swap32_posix ()
                 ));
     done
 }
-./comsub-posix5.sub: command substitution: line 38: syntax error near unexpected token `done'
-./comsub-posix5.sub: command substitution: line 38: `case x in x) ;; x) done esac)'
-./comsub-posix5.sub: command substitution: line 39: syntax error near unexpected token `done'
-./comsub-posix5.sub: command substitution: line 39: `case x in x) ;; x) done ;; esac)'
-./comsub-posix5.sub: command substitution: line 40: syntax error near unexpected token `esac'
-./comsub-posix5.sub: command substitution: line 40: `case x in x) (esac) esac)'
-bash: -c: line 1: syntax error near unexpected token `;;'
+./comsub-posix5.sub: line 37: syntax error near unexpected token `done'
+./comsub-posix5.sub: line 37: `: $(case x in x) ;; x) done esac)'
+./comsub-posix5.sub: line 38: syntax error near unexpected token `done'
+./comsub-posix5.sub: line 38: `: $(case x in x) ;; x) done ;; esac)'
+./comsub-posix5.sub: line 39: syntax error near unexpected token `esac'
+./comsub-posix5.sub: line 39: `: $(case x in x) (esac) esac)'
+bash: -c: line 1: syntax error near unexpected token `in'
 bash: -c: line 1: `: $(case x in esac|in) foo;; esac)'
-bash: -c: line 1: unexpected EOF while looking for matching `)'
-bash: -c: line 2: syntax error: unexpected end of file
+bash: -c: line 1: syntax error near unexpected token `done'
+bash: -c: line 1: `: $(case x in x) ;; x) done)'
 yes
+
+
+
 ab cde

tests/comsub-posix.tests

@@ -274,5 +274,14 @@ testing=$(
     done
 )
 
+# sanity check for empty comsubs
+echo $()
+echo $(
+)
+
+echo $(		
+
+   )
+
 # recommended to be parsed as a nested comsub instead of arithsub
 echo $(( echo ab cde ) )

tests/cprint.right

@@ -19,7 +19,7 @@ tf ()
         i=$(( i + 1 ));
     done;
     [[ -r /dev/fd/0 && -w /dev/fd/1 ]] || echo oops > /dev/null;
-    for name in $( echo 1 2 3 );
+    for name in $(echo 1 2 3);
     do
         test -r /dev/fd/$name;
     done;

tests/errors.right

@@ -54,10 +54,10 @@ umask: usage: umask [-p] [-S] [mode]
 ./errors.tests: line 177: declare: VAR: readonly variable
 ./errors.tests: line 179: declare: unset: not found
 ./errors.tests: line 182: VAR: readonly variable
-./errors.tests: command substitution: line 186: syntax error near unexpected token `)'
-./errors.tests: command substitution: line 186: ` for z in 1 2 3; do )'
-./errors.tests: command substitution: line 187: syntax error near unexpected token `done'
-./errors.tests: command substitution: line 187: ` for z in 1 2 3; done )'
+./errors.tests: line 185: syntax error near unexpected token `)'
+./errors.tests: line 185: `: $( for z in 1 2 3; do )'
+./errors.tests: line 186: syntax error near unexpected token `done'
+./errors.tests: line 186: `: $( for z in 1 2 3; done )'
 ./errors.tests: line 189: cd: HOME not set
 ./errors.tests: line 190: cd: /tmp/xyz.bash: No such file or directory
 ./errors.tests: line 192: cd: OLDPWD not set

tests/heredoc.right

@@ -67,13 +67,13 @@ qux
 bar
 qux
 abc def geh
-./heredoc3.sub: line 23: warning: here-document at line 21 delimited by end-of-file (wanted `EOF')
+./heredoc3.sub: line 20: warning: here-document at line 18 delimited by end-of-file (wanted `EOF')
 = here is the text =
-./heredoc3.sub: line 29: warning: here-document at line 27 delimited by end-of-file (wanted `EOF')
+./heredoc3.sub: line 26: warning: here-document at line 24 delimited by end-of-file (wanted `EOF')
 this paren ) is not a problem
-./heredoc3.sub: line 35: warning: here-document at line 33 delimited by end-of-file (wanted `EOF')
+./heredoc3.sub: line 32: warning: here-document at line 30 delimited by end-of-file (wanted `EOF')
 these balanced parens ( ) are not a problem
-./heredoc3.sub: line 41: warning: here-document at line 39 delimited by end-of-file (wanted `EOF')
+./heredoc3.sub: line 38: warning: here-document at line 36 delimited by end-of-file (wanted `EOF')
 quoted balanced parens \( ) are not a problem either
 more text in a subshell
 some more text in a different subshell

tests/nquote.right

@@ -66,3 +66,5 @@ argv[1] = <^?>
 0000000 esc fs gs rs us del nl
 0000007
 \q
+foo
+./nquote4.sub: line 6: quux: command not found

tests/nquote.tests

@@ -138,3 +138,4 @@ echo "$(echo $'\t\t\101\104\n\105')"
 ${THIS_SH} ./nquote1.sub
 ${THIS_SH} ./nquote2.sub
 ${THIS_SH} ./nquote3.sub
+${THIS_SH} ./nquote4.sub

tests/nquote4.sub

@@ -0,0 +1,6 @@
+# make sure aliases containing new-style single quotes are expanded correctly
+
+shopt -s expand_aliases
+alias foo=$'echo foo\n\nquux\n'
+
+foo

tests/type.right

@@ -54,12 +54,12 @@ f ()
 foo is a function
 foo () 
 { 
-    echo $(<x1)
+    echo $(< x1)
 }
 bar is a function
 bar () 
 { 
-    echo $(<x1)
+    echo $(< x1)
 }
 foo is a function
 foo ()