From 83c4ad27c7f1284ecb5db6e2f1afc0c553d5fcf3 Mon Sep 17 00:00:00 2001 From: Chet Ramey Date: Mon, 15 Mar 2021 09:30:44 -0400 Subject: [PATCH] process $'...' and $"..." in here-documents --- CWRU/CWRU.chlog | 24 ++++++++++++ MANIFEST | 1 + bashline.c | 12 +++++- make_cmd.c | 15 ++++++-- parse.y | 24 ++++++++++-- subst.c | 90 +++++++++++++++++++++++++++++++++++++++++++- subst.h | 3 ++ tests/heredoc.right | 22 ++++++++++- tests/heredoc.tests | 3 ++ tests/heredoc6.sub | 50 ++++++++++++++++++++++++ tests/nquote.right | 2 + tests/nquote.tests | 4 ++ tests/posixexp.right | 5 ++- 13 files changed, 242 insertions(+), 13 deletions(-) create mode 100644 tests/heredoc6.sub diff --git a/CWRU/CWRU.chlog b/CWRU/CWRU.chlog index 409e68bc..15c5c131 100644 --- a/CWRU/CWRU.chlog +++ b/CWRU/CWRU.chlog @@ -9826,3 +9826,27 @@ builtins/read.def - read_builtin: changes to use the readline timeout functions to implement timeouts with `read -e'; these use rl_set_timeout and sh_timer structs together + + 3/12 + ---- +subst.c + - expand_string_dollar_quote: new function, expands $'...' and $"..." + in a string for those code paths that don't expand it themselves + +subst.h + - expand_string_dollar_quote: extern declaration + +parse.y + - read_secondary_line: if $'...' or $"..." appears in the line, call + expand_string_dollar_quote to expand them. This now returns new + memory, need to change callers + +make_cmd.c + - make_here_document: account for read_secondary_line returning newly + allocated memory, free `full_line' appropriately + +bashline.c + - shell_expand_line,history_and_alias_expand_line: expand $'...' and + $"..." in the line by calling expand_string_dollar_quote, since + that happens after history expansion and before alias expansion in + normal processing diff --git a/MANIFEST b/MANIFEST index 45cd49bd..6fa77817 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1149,6 +1149,7 @@ tests/heredoc2.sub f tests/heredoc3.sub f tests/heredoc4.sub f tests/heredoc5.sub f +tests/heredoc6.sub f tests/herestr.tests f tests/herestr.right f tests/herestr1.sub f diff --git a/bashline.c b/bashline.c index 2569d4dc..aea730df 100644 --- a/bashline.c +++ b/bashline.c @@ -2764,13 +2764,17 @@ static int history_and_alias_expand_line (count, ignore) int count, ignore; { - char *new_line; + char *new_line, *t; new_line = 0; #if defined (BANG_HISTORY) new_line = history_expand_line_internal (rl_line_buffer); #endif + t = expand_string_dollar_quote (new_line ? new_line : rl_line_buffer, 0); + FREE (new_line); + new_line = t; + #if defined (ALIAS) if (new_line) { @@ -2802,7 +2806,7 @@ static int shell_expand_line (count, ignore) int count, ignore; { - char *new_line; + char *new_line, *t; WORD_LIST *expanded_string; WORD_DESC *w; @@ -2811,6 +2815,10 @@ shell_expand_line (count, ignore) new_line = history_expand_line_internal (rl_line_buffer); #endif + t = expand_string_dollar_quote (new_line ? new_line : rl_line_buffer, 0); + FREE (new_line); + new_line = t; + #if defined (ALIAS) if (new_line) { diff --git a/make_cmd.c b/make_cmd.c index 2d7ac960..f9a8d1df 100644 --- a/make_cmd.c +++ b/make_cmd.c @@ -575,7 +575,7 @@ make_here_document (temp, lineno) kill_leading = temp->instruction == r_deblank_reading_until; - document = (char *)NULL; + full_line = document = (char *)NULL; document_index = document_size = 0; /* Quote removal is the only expansion performed on the delimiter @@ -628,17 +628,20 @@ make_here_document (temp, lineno) check the word before stripping the whitespace. This is a hack, though. */ if (STREQN (line, redir_word, redir_len) && line[redir_len] == '\n') - goto document_done; + break; while (*line == '\t') line++; } if (*line == 0) - continue; + { + free (full_line); + continue; + } if (STREQN (line, redir_word, redir_len) && line[redir_len] == '\n') - goto document_done; + break; len = strlen (line); if (len + document_index >= document_size) @@ -651,11 +654,15 @@ make_here_document (temp, lineno) being an empty string before the call to strlen. */ FASTCOPY (line, document + document_index, len); document_index += len; + + free (full_line); } if (full_line == 0) internal_warning (_("here-document at line %d delimited by end-of-file (wanted `%s')"), lineno, redir_word); + FREE (full_line); + document_done: if (document) document[document_index] = '\0'; diff --git a/parse.y b/parse.y index 70564241..e95eb51a 100644 --- a/parse.y +++ b/parse.y @@ -2116,12 +2116,14 @@ read_a_line (remove_quoted_newline) the secondary prompt. This is used to read the lines of a here document. REMOVE_QUOTED_NEWLINE is non-zero if we should remove newlines quoted with backslashes while reading the line. It is - non-zero unless the delimiter of the here document was quoted. */ + non-zero unless the delimiter of the here document was quoted. + If it is zero, we don't perform $'...' and $"..." expansion because + we treat the lines as if they are between double quotes. */ char * read_secondary_line (remove_quoted_newline) int remove_quoted_newline; { - char *ret; + char *ret, *t; int n, c; prompt_string_pointer = &ps2_prompt; @@ -2141,7 +2143,23 @@ read_secondary_line (remove_quoted_newline) maybe_add_history (ret); } #endif /* HISTORY */ - return ret; + if (ret == 0) + return ret; + if (remove_quoted_newline == 0) + return (savestring (ret)); + + t = ret; + while (t = strchr (t, '$')) + { + if (t[1] == '\'' || t[1] == '"') + break; + else + t++; + } + if (t == 0) + return (savestring (ret)); + t = expand_string_dollar_quote (ret, 1); + return t; } /* **************************************************************** */ diff --git a/subst.c b/subst.c index b82ada60..64979884 100644 --- a/subst.c +++ b/subst.c @@ -3721,6 +3721,95 @@ cond_expand_word (w, special) } #endif +/* Expand $'...' and $"..." in a string for code paths that don't do it. The + FLAGS argument is 1 if this function should treat CTLESC as a quote + character (e.g., for here-documents) or not (e.g., for shell_expand_line). */ +char * +expand_string_dollar_quote (string, flags) + char *string; + int flags; +{ + size_t slen; + int sindex, c, translen, retind, retsize, peekc, news; + char *ret, *trans, *send, *t; + DECLARE_MBSTATE; + + retsize = slen + 1; + ret = xmalloc (retsize); + retind = 0; + + slen = strlen (string); + send = string + slen; + sindex = 0; + + while (c = string[sindex]) + { + switch (c) + { + default: + RESIZE_MALLOCED_BUFFER (ret, retind, locale_mb_cur_max + 1, retsize, 64); + COPY_CHAR_I (ret, retind, string, send, sindex); + break; + + case '\\': + RESIZE_MALLOCED_BUFFER (ret, retind, locale_mb_cur_max + 2, retsize, 64); + ret[retind++] = string[sindex++]; + + if (string[sindex]) + COPY_CHAR_I (ret, retind, string, send, sindex); + break; + + case CTLESC: + RESIZE_MALLOCED_BUFFER (ret, retind, locale_mb_cur_max + 2, retsize, 64); + if (flags) + ret[retind++] = string[sindex++]; + if (string[sindex]) + COPY_CHAR_I (ret, retind, string, send, sindex); + break; + + case '$': + peekc = string[++sindex]; + if (peekc != '\'' && peekc != '"') + { + RESIZE_MALLOCED_BUFFER (ret, retind, 1, retsize, 16); + ret[retind++] = c; + break; + } + if (peekc == '\'') + { + /* We overload SX_COMPLETE below */ + news = skip_single_quoted (string, slen, ++sindex, SX_COMPLETE); + t = substring (string, sindex, news - 1); + trans = ansiexpand (t, 0, news-sindex-1, &translen); + free (t); + t = sh_single_quote (trans); + sindex = news; + } + else + { + news = ++sindex; + t = string_extract_double_quoted (string, &news, SX_COMPLETE); + trans = localeexpand (t, 0, news-sindex, 0, &translen); + free (t); + t = sh_mkdoublequoted (trans, translen, 0); + sindex = news; + } + free (trans); + trans = t; + translen = strlen (trans); + + RESIZE_MALLOCED_BUFFER (ret, retind, translen + 1, retsize, 128); + strcpy (ret + retind, trans); + retind += translen; + FREE (trans); + break; + } + } + + ret[retind] = 0; + return ret; +} + /* Call expand_word_internal to expand W and handle error returns. A convenience function for functions that don't want to handle any errors or free any memory before aborting. */ @@ -3845,7 +3934,6 @@ expand_string_assignment (string, quoted) return (value); } - /* Expand one of the PS? prompt strings. This is a sort of combination of expand_string_unsplit and expand_string_internal, but returns the passed string when an error occurs. Might want to trap other calls diff --git a/subst.h b/subst.h index 13476515..6a69c350 100644 --- a/subst.h +++ b/subst.h @@ -187,6 +187,9 @@ extern char *expand_assignment_string_to_string PARAMS((char *, int)); /* Expand an arithmetic expression string */ extern char *expand_arith_string PARAMS((char *, int)); +/* Expand $'...' and $"..." in a string for code paths that do not. */ +extern char *expand_string_dollar_quote PARAMS((char *, int)); + /* De-quote quoted characters in STRING. */ extern char *dequote_string PARAMS((char *)); diff --git a/tests/heredoc.right b/tests/heredoc.right index f6e53bea..8dcb7d83 100644 --- a/tests/heredoc.right +++ b/tests/heredoc.right @@ -100,7 +100,27 @@ argv[2] = argv[1] = argv[2] = argv[3] = +1: OK +2: OK +3: OK +4: OK +5: OK +6: OK +7: OK +1: OK +2: OK +3: OK +4: OK +5: OK +5: OK +1: ${x#$'no\t'} +2: O${x#$'no\t'O} +3: ${x#n$'o\t'} +4: ${x#'no '} +5: ${x#$pat} +6: ${y#$'not'} +7: ${y#'not'} comsub here-string -./heredoc.tests: line 149: warning: here-document at line 147 delimited by end-of-file (wanted `EOF') +./heredoc.tests: line 152: warning: here-document at line 150 delimited by end-of-file (wanted `EOF') hi there diff --git a/tests/heredoc.tests b/tests/heredoc.tests index 430302f5..36c005af 100644 --- a/tests/heredoc.tests +++ b/tests/heredoc.tests @@ -137,6 +137,9 @@ ${THIS_SH} ./heredoc4.sub # heredoc tests that use different size documents to test pipe implementation ${THIS_SH} ./heredoc5.sub +# test $'...' and $"..." quoted strings in here-documents +${THIS_SH} ./heredoc6.sub + echo $( cat <<< "comsub here-string" ) diff --git a/tests/heredoc6.sub b/tests/heredoc6.sub new file mode 100644 index 00000000..1d5fff09 --- /dev/null +++ b/tests/heredoc6.sub @@ -0,0 +1,50 @@ +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# test $'...' and $"..." strings in here documents (problem through bash-5.1) + +pat=$'no\t' +x=$'no\tOK' +y=notOK + +cat < ;foo argv[1] = <^I> argv[1] = <'A^IB'> +argv[1] = +argv[1] = <$'a\tb\tc'> hello' world hello world! hello' world! diff --git a/tests/nquote.tests b/tests/nquote.tests index 20d6415d..958f5c75 100644 --- a/tests/nquote.tests +++ b/tests/nquote.tests @@ -128,6 +128,10 @@ unset mytab recho "${mytab:-$'\t'}" recho "$( args $'A\tB' )" +# tests for $'...' not being expanded when inside double quotes +recho $'a\tb\tc' +recho "$'a\tb\tc'" + ${THIS_SH} ./nquote1.sub ${THIS_SH} ./nquote2.sub ${THIS_SH} ./nquote3.sub diff --git a/tests/posixexp.right b/tests/posixexp.right index e6bdbf4b..c952521c 100644 --- a/tests/posixexp.right +++ b/tests/posixexp.right @@ -275,10 +275,11 @@ argv[2] = [ abc def ghi jkl / abc def ghi jkl ] [ abc def ghi jkl ] [ abc def ghi jkl / abc def ghi jkl / abc def ghi jkl ] -5: notOK +5: OK OK OK -5: $'not\ttoo\nbad' +5: 'not too +bad' "A" A argv[1] = <"A">