From f188aa6a013e89d421e39354086eed513652b492 Mon Sep 17 00:00:00 2001 From: Chet Ramey Date: Mon, 4 Oct 2021 15:30:21 -0400 Subject: [PATCH] enable support for using `&' in the pattern substitution replacement string --- CWRU/CWRU.chlog | 58 +++++++- MANIFEST | 1 + builtins/evalstring.c | 4 - builtins/mapfile.def | 2 +- doc/bash.0 | 47 ++++-- doc/bash.1 | 46 ++++-- doc/bashref.info | 337 +++++++++++++++++++++++++----------------- doc/bashref.texi | 100 ++++++++++++- doc/builtins.1 | 10 +- doc/version.texi | 6 +- execute_cmd.c | 10 +- lib/readline/rltty.c | 2 + shell.h | 1 + subst.c | 30 ++-- tests/RUN-ONE-TEST | 2 +- tests/mapfile.right | 1 + tests/mapfile.tests | 7 +- tests/new-exp.right | 22 ++- tests/new-exp.tests | 4 + tests/new-exp10.sub | 4 +- tests/new-exp16.sub | 31 ++++ trap.c | 2 +- 22 files changed, 518 insertions(+), 209 deletions(-) create mode 100644 tests/new-exp16.sub diff --git a/CWRU/CWRU.chlog b/CWRU/CWRU.chlog index 6c3f7d9c..0f64c517 100644 --- a/CWRU/CWRU.chlog +++ b/CWRU/CWRU.chlog @@ -2126,4 +2126,60 @@ arrayfunc.c indexed array values, we need to quote CTLESC characters, because that's how they come out of the parser and how the assignment statement code expects to see them. - Fixes https://savannah.gnu.org/support/index.php?110538 + Fixes https://savannah.gnu.org/support/index.php?110538 + + 9/29 + ---- +subst.c + - parameter_brace_transform: invalid transformation operators are now + fatal errors in non-interactive shells, as with the other word + expansions. Reported by Martin Schulte in + https://lists.gnu.org/archive/html/bug-bash/2020-10/msg00026.html + +execute_cmd.c + - execute_disk_command: if we're optimizing out the fork (nofork) and + not directly in a pipeline (pipe_in == pipe_out == NO_PIPE), only + modify shell_level if subshell_environment says we're not already in + a pipeline. Reported by Paul Smith 10/11/2020 + against GNU make + +evalstring.c + - should_suppress_fork: remove #if 1 for code that tries to suppress + the fork in a process substitution subshell + + 9/30 + ---- +builtins/mapfile.def + - do_chop: make sure we're comparing unsigned chars when checking + whether the delim is the last character on the line. Reported by + Greg Wooledge + + 10/1 + ---- +lib/readline/rltty.c + - rl_deprep_terminal: if we're not echoing to the terminal + (_rl_echoing_p == 0), and we just output the bracketed paste end + sequence, output a newline to compensate for the \r at the end of + BRACK_PASTE_FINI, since redisplay didn't do it for us. Reported by + Siteshwar Vashisht + +shell.h + - MATCH_EXPREP: new matching flag, understood only by pattern + substitution; means to expand unquoted `&' in the replacement + STRING to the match of PATTERN + +subst.c + - shouldexp_replacement: uncommented + - pat_subst: we expand & in the replacement string if MATCH_EXPREP + appears in MFLAGS + - parameter_brace_patsub: push call to shouldexp_replacement out here, + after the replacement string is expanded; set MATCH_EXPREP if there + is an unquoted `&' (by backslash) in the expanded replacement + string + +doc/{bash.1,bashref.texi} + - pattern substitution: overhauled the description, moved each of the + possible forms to be tags in the tagged paragraph. The description + now specifies the expansions that the replacement string undergoes + - pattern substitution: documented new behavior of unquoted & in the + replacement string diff --git a/MANIFEST b/MANIFEST index c51bb3b0..2073e987 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1258,6 +1258,7 @@ tests/new-exp12.sub f tests/new-exp13.sub f tests/new-exp14.sub f tests/new-exp15.sub f +tests/new-exp16.sub f tests/new-exp.right f tests/nquote.tests f tests/nquote.right f diff --git a/builtins/evalstring.c b/builtins/evalstring.c index 296e61bd..4fd2c567 100644 --- a/builtins/evalstring.c +++ b/builtins/evalstring.c @@ -99,11 +99,7 @@ should_suppress_fork (command) signal_is_trapped (EXIT_TRAP) == 0 && signal_is_trapped (ERROR_TRAP) == 0 && any_signals_trapped () < 0 && -#if 1 /* TAG: bash-5.2 */ (subshell || (command->redirects == 0 && command->value.Simple->redirects == 0)) && -#else - command->redirects == 0 && command->value.Simple->redirects == 0 && -#endif ((command->flags & CMD_TIME_PIPELINE) == 0) && ((command->flags & CMD_INVERT_RETURN) == 0)); } diff --git a/builtins/mapfile.def b/builtins/mapfile.def index 62db816c..c17fc5c5 100644 --- a/builtins/mapfile.def +++ b/builtins/mapfile.def @@ -141,7 +141,7 @@ do_chop(line, delim) int length; length = strlen (line); - if (length && line[length-1] == delim) + if (length && (unsigned char)line[length-1] == delim) line[length-1] = '\0'; } diff --git a/doc/bash.0 b/doc/bash.0 index 35b575cc..701e667c 100644 --- a/doc/bash.0 +++ b/doc/bash.0 @@ -1827,17 +1827,32 @@ EEXXPPAANNSSIIOONN turn, and the expansion is the resultant list. ${_p_a_r_a_m_e_t_e_r//_p_a_t_t_e_r_n//_s_t_r_i_n_g} + ${_p_a_r_a_m_e_t_e_r////_p_a_t_t_e_r_n//_s_t_r_i_n_g} + ${_p_a_r_a_m_e_t_e_r//##_p_a_t_t_e_r_n//_s_t_r_i_n_g} + ${_p_a_r_a_m_e_t_e_r//%%_p_a_t_t_e_r_n//_s_t_r_i_n_g} PPaatttteerrnn ssuubbssttiittuuttiioonn. The _p_a_t_t_e_r_n is expanded to produce a pat- - tern just as in pathname expansion, _P_a_r_a_m_e_t_e_r is expanded and + tern just as in pathname expansion. _P_a_r_a_m_e_t_e_r is expanded and the longest match of _p_a_t_t_e_r_n against its value is replaced with - _s_t_r_i_n_g. The match is performed using the rules described under - PPaatttteerrnn MMaattcchhiinngg below. If _p_a_t_t_e_r_n begins with //, all matches - of _p_a_t_t_e_r_n are replaced with _s_t_r_i_n_g. Normally only the first - match is replaced. If _p_a_t_t_e_r_n begins with ##, it must match at - the beginning of the expanded value of _p_a_r_a_m_e_t_e_r. If _p_a_t_t_e_r_n - begins with %%, it must match at the end of the expanded value of - _p_a_r_a_m_e_t_e_r. If _s_t_r_i_n_g is null, matches of _p_a_t_t_e_r_n are deleted - and the // following _p_a_t_t_e_r_n may be omitted. If the nnooccaasseemmaattcchh + _s_t_r_i_n_g. _s_t_r_i_n_g undergoes tilde expansion, parameter and vari- + able expansion, arithmetic expansion, command and process sub- + stitution, and quote removal. The match is performed using the + rules described under PPaatttteerrnn MMaattcchhiinngg below. In the first form + above, only the first match is replaced. If there are two + slashes separating _p_a_r_a_m_e_t_e_r and _p_a_t_t_e_r_n (the second form + above), all matches of _p_a_t_t_e_r_n are replaced with _s_t_r_i_n_g. If + _p_a_t_t_e_r_n is preceded by ## (the third form above), it must match + at the beginning of the expanded value of _p_a_r_a_m_e_t_e_r. If _p_a_t_t_e_r_n + is preceded by %% (the fourth form above), it must match at the + end of the expanded value of _p_a_r_a_m_e_t_e_r. If the expansion of + _s_t_r_i_n_g is null, matches of _p_a_t_t_e_r_n are deleted. If _s_t_r_i_n_g is + null, matches of _p_a_t_t_e_r_n are deleted and the // following _p_a_t_t_e_r_n + may be omitted. Any unquoted instances of && in _s_t_r_i_n_g are re- + placed with the matching portion of _p_a_t_t_e_r_n. Backslash is used + to quote && in _s_t_r_i_n_g; users should take care if _s_t_r_i_n_g is dou- + ble-quoted to avoid unwanted interactions between the backslash + and double-quoting. Pattern substitution performs the check for + && before expanding _s_t_r_i_n_g to catch backslashes escaping the && + before they are removed by the expansion. If the nnooccaasseemmaattcchh shell option is enabled, the match is performed without regard to the case of alphabetic characters. If _p_a_r_a_m_e_t_e_r is @@ or **, the substitution operation is applied to each positional parame- @@ -2695,11 +2710,13 @@ SSIIMMPPLLEE CCOOMMMMAANNDD EEXXPPAANNSSIIOONN able. If no command name results, the variable assignments affect the current - shell environment. Otherwise, the variables are added to the environ- - ment of the executed command and do not affect the current shell envi- - ronment. If any of the assignments attempts to assign a value to a - readonly variable, an error occurs, and the command exits with a non- - zero status. + shell environment. In the case of such a command (one that consists + only of assignment statements and redirections), assignment statements + are performed before redirections. Otherwise, the variables are added + to the environment of the executed command and do not affect the cur- + rent shell environment. If any of the assignments attempts to assign a + value to a readonly variable, an error occurs, and the command exits + with a non-zero status. If no command name results, redirections are performed, but do not af- fect the current shell environment. A redirection error causes the @@ -6517,4 +6534,4 @@ BBUUGGSS -GNU Bash 5.1 2021 August 23 BASH(1) +GNU Bash 5.1 2021 October 1 BASH(1) diff --git a/doc/bash.1 b/doc/bash.1 index 492cceba..216b1b44 100644 --- a/doc/bash.1 +++ b/doc/bash.1 @@ -5,12 +5,12 @@ .\" Case Western Reserve University .\" chet.ramey@case.edu .\" -.\" Last Change: Mon Aug 23 10:08:28 EDT 2021 +.\" Last Change: Fri Oct 1 14:20:30 EDT 2021 .\" .\" bash_builtins, strip all but Built-Ins section .if \n(zZ=1 .ig zZ .if \n(zY=1 .ig zY -.TH BASH 1 "2021 August 23" "GNU Bash 5.1" +.TH BASH 1 "2021 October 1" "GNU Bash 5.1" .\" .\" There's some problem with having a `@' .\" in a tagged paragraph with the BSD man macros. @@ -3336,22 +3336,48 @@ the pattern removal operation is applied to each member of the array in turn, and the expansion is the resultant list. .TP ${\fIparameter\fP\fB/\fP\fIpattern\fP\fB/\fP\fIstring\fP} +.PD 0 +.TP +${\fIparameter\fP\fB//\fP\fIpattern\fP\fB/\fP\fIstring\fP} +.TP +${\fIparameter\fP\fB/#\fP\fIpattern\fP\fB/\fP\fIstring\fP} +.TP +${\fIparameter\fP\fB/%\fP\fIpattern\fP\fB/\fP\fIstring\fP} +.PD \fBPattern substitution\fP. The \fIpattern\fP is expanded to produce a pattern just as in -pathname expansion, +pathname expansion. \fIParameter\fP is expanded and the longest match of \fIpattern\fP against its value is replaced with \fIstring\fP. +\fIstring\fP undergoes tilde expansion, parameter and variable expansion, +arithmetic expansion, command and process substitution, and quote removal. The match is performed using the rules described under .B Pattern Matching below. -If \fIpattern\fP begins with \fB/\fP, all matches of \fIpattern\fP are -replaced with \fIstring\fP. Normally only the first match is replaced. -If \fIpattern\fP begins with \fB#\fP, it must match at the beginning -of the expanded value of \fIparameter\fP. -If \fIpattern\fP begins with \fB%\fP, it must match at the end -of the expanded value of \fIparameter\fP. -If \fIstring\fP is null, matches of \fIpattern\fP are deleted +In the first form above, only the first match is replaced. +If there are two slashes separating \fIparameter\fP and \fIpattern\fP +(the second form above), all matches of \fIpattern\fP are +replaced with \fIstring\fP. +If \fIpattern\fP is preceded by \fB#\fP (the third form above), +it must match at the beginning of the expanded value of \fIparameter\fP. +If \fIpattern\fP is preceded by \fB%\fP (the fourth form above), +it must match at the end of the expanded value of \fIparameter\fP. +If the expansion of \fIstring\fP is null, +matches of \fIpattern\fP are deleted. +If \fIstring\fP is null, +matches of \fIpattern\fP are deleted and the \fB/\fP following \fIpattern\fP may be omitted. +Any unquoted instances of \fB&\fP in \fIstring\fP are replaced with the +matching portion of \fIpattern\fP. +Backslash is used to quote \fB&\fP in \fIstring\fP; the backslash is removed +in order to permit a literal \fB&\fP in the replacement string. +Users should take care +if \fIstring\fP is double-quoted to avoid unwanted interactions between +the backslash and double-quoting. +Pattern substitution performs the check for \fB&\fP after expanding +\fIstring\fP; shell programmers should quote backslashes intended to escape +the \fB&\fP and inhibit replacement so they survive any quote removal +performed by the expansion of \fIstring\fP. If the .B nocasematch shell option is enabled, the match is performed without regard to the case diff --git a/doc/bashref.info b/doc/bashref.info index fee550b7..8ff13d7f 100644 --- a/doc/bashref.info +++ b/doc/bashref.info @@ -1,10 +1,10 @@ -This is bashref.info, produced by makeinfo version 6.7 from +This is bashref.info, produced by makeinfo version 6.8 from bashref.texi. This text is a brief description of the features that are present in the -Bash shell (version 5.1, 23 August 2021). +Bash shell (version 5.1, 1 October 2021). - This is Edition 5.1, last updated 23 August 2021, of 'The GNU Bash + This is Edition 5.1, last updated 1 October 2021, of 'The GNU Bash Reference Manual', for 'Bash', Version 5.1. Copyright (C) 1988-2021 Free Software Foundation, Inc. @@ -27,10 +27,10 @@ Bash Features ************* This text is a brief description of the features that are present in the -Bash shell (version 5.1, 23 August 2021). The Bash home page is +Bash shell (version 5.1, 1 October 2021). The Bash home page is . - This is Edition 5.1, last updated 23 August 2021, of 'The GNU Bash + This is Edition 5.1, last updated 1 October 2021, of 'The GNU Bash Reference Manual', for 'Bash', Version 5.1. Bash contains features that appear in other popular shells, and some @@ -1831,22 +1831,39 @@ omitted, the operator tests only for existence. If PARAMETER is unset or null, the expansion of WORD is substituted. Otherwise, the value of PARAMETER is substituted. + $ v=123 + $ echo ${v-unset} + 123 + '${PARAMETER:=WORD}' If PARAMETER is unset or null, the expansion of WORD is assigned to PARAMETER. The value of PARAMETER is then substituted. Positional parameters and special parameters may not be assigned to in this way. + $ var= + $ : ${var:=DEFAULT} + $ echo $var + DEFAULT + '${PARAMETER:?WORD}' If PARAMETER is null or unset, the expansion of WORD (or a message to that effect if WORD is not present) is written to the standard error and the shell, if it is not interactive, exits. Otherwise, the value of PARAMETER is substituted. + $ var= + $ : ${var:?var is unset or null} + bash: var: var is unset or null + '${PARAMETER:+WORD}' If PARAMETER is null or unset, nothing is substituted, otherwise the expansion of WORD is substituted. + $ var=123 + $ echo ${var:+var is set and not null} + var is set and not null + '${PARAMETER:OFFSET}' '${PARAMETER:OFFSET:LENGTH}' This is referred to as Substring Expansion. It expands to up to @@ -2037,26 +2054,66 @@ omitted, the operator tests only for existence. array in turn, and the expansion is the resultant list. '${PARAMETER/PATTERN/STRING}' - +'${PARAMETER//PATTERN/STRING}' +'${PARAMETER/#PATTERN/STRING}' +'${PARAMETER/%PATTERN/STRING}' The PATTERN is expanded to produce a pattern just as in filename expansion. PARAMETER is expanded and the longest match of PATTERN - against its value is replaced with STRING. The match is performed - according to the rules described below (*note Pattern Matching::). - If PATTERN begins with '/', all matches of PATTERN are replaced - with STRING. Normally only the first match is replaced. If - PATTERN begins with '#', it must match at the beginning of the - expanded value of PARAMETER. If PATTERN begins with '%', it must - match at the end of the expanded value of PARAMETER. If STRING is - null, matches of PATTERN are deleted and the '/' following PATTERN - may be omitted. If the 'nocasematch' shell option (see the - description of 'shopt' in *note The Shopt Builtin::) is enabled, - the match is performed without regard to the case of alphabetic - characters. If PARAMETER is '@' or '*', the substitution operation - is applied to each positional parameter in turn, and the expansion - is the resultant list. If PARAMETER is an array variable - subscripted with '@' or '*', the substitution operation is applied - to each member of the array in turn, and the expansion is the - resultant list. + against its value is replaced with STRING. STRING undergoes tilde + expansion, parameter and variable expansion, arithmetic expansion, + command and process substitution, and quote removal. The match is + performed according to the rules described below (*note Pattern + Matching::). + + In the first form above, only the first match is replaced. If + there are two slashes separating PARAMETER and PATTERN (the second + form above), all matches of PATTERN are replaced with STRING. If + PATTERN is preceded by '#' (the third form above), it must match at + the beginning of the expanded value of PARAMETER. If PATTERN is + preceded by '%' (the fourth form above), it must match at the end + of the expanded value of PARAMETER. If the expansion of STRING is + null, matches of PATTERN are deleted. If STRING is null, matches + of PATTERN are deleted and the '/' following PATTERN may be + omitted. + + Any unquoted instances of '&' in STRING are replaced with the + matching portion of PATTERN. This is intended to duplicate a + common 'sed' idiom. Backslash is used to quote '&' in STRING; + users should take care if STRING is double-quoted to avoid unwanted + interactions between the backslash and double-quoting. For + instance, + + var=abcdef + echo ${var/abc/& } + echo "${var/abc/& }" + echo ${var/abc/"& "} + + will display three lines of "abc def", while + + var=abcdef + echo ${var/abc/\& } + echo "${var/abc/\& }" + echo ${var/abc/"\& "} + + will display two lines of "& def" and a third line of "\& def". + This is because '&' is not one of the characters for which + backslash is special in double quotes, and the expansion of STRING + is performed in a context that doesn't take the enclosing double + quotes into account. It should rarely be necessary to enclose only + STRING in double quotes. + + Pattern substitution performs the check for '&' before expanding + STRING to catch backslashes escaping the '&' before they are + removed by the expansion. + + If the 'nocasematch' shell option (see the description of 'shopt' + in *note The Shopt Builtin::) is enabled, the match is performed + without regard to the case of alphabetic characters. If PARAMETER + is '@' or '*', the substitution operation is applied to each + positional parameter in turn, and the expansion is the resultant + list. If PARAMETER is an array variable subscripted with '@' or + '*', the substitution operation is applied to each member of the + array in turn, and the expansion is the resultant list. '${PARAMETER^PATTERN}' '${PARAMETER^^PATTERN}' @@ -2067,18 +2124,22 @@ omitted, the operator tests only for existence. filename expansion. Each character in the expanded value of PARAMETER is tested against PATTERN, and, if it matches the pattern, its case is converted. The pattern should not attempt to - match more than one character. The '^' operator converts lowercase - letters matching PATTERN to uppercase; the ',' operator converts - matching uppercase letters to lowercase. The '^^' and ',,' - expansions convert each matched character in the expanded value; - the '^' and ',' expansions match and convert only the first - character in the expanded value. If PATTERN is omitted, it is - treated like a '?', which matches every character. If PARAMETER is - '@' or '*', the case modification operation is applied to each - positional parameter in turn, and the expansion is the resultant - list. If PARAMETER is an array variable subscripted with '@' or - '*', the case modification operation is applied to each member of - the array in turn, and the expansion is the resultant list. + match more than one character. + + The '^' operator converts lowercase letters matching PATTERN to + uppercase; the ',' operator converts matching uppercase letters to + lowercase. The '^^' and ',,' expansions convert each matched + character in the expanded value; the '^' and ',' expansions match + and convert only the first character in the expanded value. If + PATTERN is omitted, it is treated like a '?', which matches every + character. + + If PARAMETER is '@' or '*', the case modification operation is + applied to each positional parameter in turn, and the expansion is + the resultant list. If PARAMETER is an array variable subscripted + with '@' or '*', the case modification operation is applied to each + member of the array in turn, and the expansion is the resultant + list. '${PARAMETER@OPERATOR}' The expansion is either a transformation of the value of PARAMETER @@ -2697,11 +2758,13 @@ following order. expansion, and quote removal before being assigned to the variable. If no command name results, the variable assignments affect the -current shell environment. Otherwise, the variables are added to the -environment of the executed command and do not affect the current shell -environment. If any of the assignments attempts to assign a value to a -readonly variable, an error occurs, and the command exits with a -non-zero status. +current shell environment. In the case of such a command (one that +consists only of assignment statements and redirections), assignment +statements are performed before redirections. Otherwise, the variables +are added to the environment of the executed command and do not affect +the current shell environment. If any of the assignments attempts to +assign a value to a readonly variable, an error occurs, and the command +exits with a non-zero status. If no command name results, redirections are performed, but do not affect the current shell environment. A redirection error causes the @@ -12264,103 +12327,103 @@ Node: Shell Expansions69743 Node: Brace Expansion71870 Node: Tilde Expansion74604 Node: Shell Parameter Expansion77225 -Node: Command Substitution92528 -Node: Arithmetic Expansion93883 -Node: Process Substitution94851 -Node: Word Splitting95971 -Node: Filename Expansion97915 -Node: Pattern Matching100515 -Node: Quote Removal105123 -Node: Redirections105418 -Node: Executing Commands115078 -Node: Simple Command Expansion115748 -Node: Command Search and Execution117702 -Node: Command Execution Environment120080 -Node: Environment123115 -Node: Exit Status124778 -Node: Signals126562 -Node: Shell Scripts128529 -Node: Shell Builtin Commands131556 -Node: Bourne Shell Builtins133594 -Node: Bash Builtins155055 -Node: Modifying Shell Behavior185872 -Node: The Set Builtin186217 -Node: The Shopt Builtin196630 -Node: Special Builtins212059 -Node: Shell Variables213038 -Node: Bourne Shell Variables213475 -Node: Bash Variables215579 -Node: Bash Features248394 -Node: Invoking Bash249407 -Node: Bash Startup Files255420 -Node: Interactive Shells260523 -Node: What is an Interactive Shell?260933 -Node: Is this Shell Interactive?261582 -Node: Interactive Shell Behavior262397 -Node: Bash Conditional Expressions265910 -Node: Shell Arithmetic270552 -Node: Aliases273496 -Node: Arrays276109 -Node: The Directory Stack282118 -Node: Directory Stack Builtins282902 -Node: Controlling the Prompt287162 -Node: The Restricted Shell290127 -Node: Bash POSIX Mode292724 -Node: Shell Compatibility Mode303997 -Node: Job Control310653 -Node: Job Control Basics311113 -Node: Job Control Builtins316115 -Node: Job Control Variables321515 -Node: Command Line Editing322671 -Node: Introduction and Notation324342 -Node: Readline Interaction325965 -Node: Readline Bare Essentials327156 -Node: Readline Movement Commands328939 -Node: Readline Killing Commands329899 -Node: Readline Arguments331817 -Node: Searching332861 -Node: Readline Init File335047 -Node: Readline Init File Syntax336308 -Node: Conditional Init Constructs357588 -Node: Sample Init File361784 -Node: Bindable Readline Commands364908 -Node: Commands For Moving366112 -Node: Commands For History368163 -Node: Commands For Text373157 -Node: Commands For Killing376806 -Node: Numeric Arguments379839 -Node: Commands For Completion380978 -Node: Keyboard Macros385169 -Node: Miscellaneous Commands385856 -Node: Readline vi Mode391795 -Node: Programmable Completion392702 -Node: Programmable Completion Builtins400482 -Node: A Programmable Completion Example411177 -Node: Using History Interactively416424 -Node: Bash History Facilities417108 -Node: Bash History Builtins420113 -Node: History Interaction425121 -Node: Event Designators428741 -Node: Word Designators430095 -Node: Modifiers431855 -Node: Installing Bash433666 -Node: Basic Installation434803 -Node: Compilers and Options438061 -Node: Compiling For Multiple Architectures438802 -Node: Installation Names440495 -Node: Specifying the System Type441313 -Node: Sharing Defaults442029 -Node: Operation Controls442702 -Node: Optional Features443660 -Node: Reporting Bugs454460 -Node: Major Differences From The Bourne Shell455735 -Node: GNU Free Documentation License472585 -Node: Indexes497762 -Node: Builtin Index498216 -Node: Reserved Word Index505043 -Node: Variable Index507491 -Node: Function Index523983 -Node: Concept Index537767 +Node: Command Substitution94537 +Node: Arithmetic Expansion95892 +Node: Process Substitution96860 +Node: Word Splitting97980 +Node: Filename Expansion99924 +Node: Pattern Matching102524 +Node: Quote Removal107132 +Node: Redirections107427 +Node: Executing Commands117087 +Node: Simple Command Expansion117757 +Node: Command Search and Execution119867 +Node: Command Execution Environment122245 +Node: Environment125280 +Node: Exit Status126943 +Node: Signals128727 +Node: Shell Scripts130694 +Node: Shell Builtin Commands133721 +Node: Bourne Shell Builtins135759 +Node: Bash Builtins157220 +Node: Modifying Shell Behavior188037 +Node: The Set Builtin188382 +Node: The Shopt Builtin198795 +Node: Special Builtins214224 +Node: Shell Variables215203 +Node: Bourne Shell Variables215640 +Node: Bash Variables217744 +Node: Bash Features250559 +Node: Invoking Bash251572 +Node: Bash Startup Files257585 +Node: Interactive Shells262688 +Node: What is an Interactive Shell?263098 +Node: Is this Shell Interactive?263747 +Node: Interactive Shell Behavior264562 +Node: Bash Conditional Expressions268075 +Node: Shell Arithmetic272717 +Node: Aliases275661 +Node: Arrays278274 +Node: The Directory Stack284283 +Node: Directory Stack Builtins285067 +Node: Controlling the Prompt289327 +Node: The Restricted Shell292292 +Node: Bash POSIX Mode294889 +Node: Shell Compatibility Mode306162 +Node: Job Control312818 +Node: Job Control Basics313278 +Node: Job Control Builtins318280 +Node: Job Control Variables323680 +Node: Command Line Editing324836 +Node: Introduction and Notation326507 +Node: Readline Interaction328130 +Node: Readline Bare Essentials329321 +Node: Readline Movement Commands331104 +Node: Readline Killing Commands332064 +Node: Readline Arguments333982 +Node: Searching335026 +Node: Readline Init File337212 +Node: Readline Init File Syntax338473 +Node: Conditional Init Constructs359753 +Node: Sample Init File363949 +Node: Bindable Readline Commands367073 +Node: Commands For Moving368277 +Node: Commands For History370328 +Node: Commands For Text375322 +Node: Commands For Killing378971 +Node: Numeric Arguments382004 +Node: Commands For Completion383143 +Node: Keyboard Macros387334 +Node: Miscellaneous Commands388021 +Node: Readline vi Mode393960 +Node: Programmable Completion394867 +Node: Programmable Completion Builtins402647 +Node: A Programmable Completion Example413342 +Node: Using History Interactively418589 +Node: Bash History Facilities419273 +Node: Bash History Builtins422278 +Node: History Interaction427286 +Node: Event Designators430906 +Node: Word Designators432260 +Node: Modifiers434020 +Node: Installing Bash435831 +Node: Basic Installation436968 +Node: Compilers and Options440226 +Node: Compiling For Multiple Architectures440967 +Node: Installation Names442660 +Node: Specifying the System Type443478 +Node: Sharing Defaults444194 +Node: Operation Controls444867 +Node: Optional Features445825 +Node: Reporting Bugs456625 +Node: Major Differences From The Bourne Shell457900 +Node: GNU Free Documentation License474750 +Node: Indexes499927 +Node: Builtin Index500381 +Node: Reserved Word Index507208 +Node: Variable Index509656 +Node: Function Index526148 +Node: Concept Index539932  End Tag Table diff --git a/doc/bashref.texi b/doc/bashref.texi index 1471526a..db9f2798 100644 --- a/doc/bashref.texi +++ b/doc/bashref.texi @@ -2156,6 +2156,12 @@ If @var{parameter} is unset or null, the expansion of @var{word} is substituted. Otherwise, the value of @var{parameter} is substituted. +@example +$ v=123 +$ echo $@{v-unset@} +123 +@end example + @item $@{@var{parameter}:=@var{word}@} If @var{parameter} is unset or null, the expansion of @var{word} @@ -2164,6 +2170,13 @@ The value of @var{parameter} is then substituted. Positional parameters and special parameters may not be assigned to in this way. +@example +$ var= +$ : $@{var:=DEFAULT@} +$ echo $var +DEFAULT +@end example + @item $@{@var{parameter}:?@var{word}@} If @var{parameter} is null or unset, the expansion of @var{word} (or a message @@ -2172,11 +2185,23 @@ is not present) is written to the standard error and the shell, if it is not interactive, exits. Otherwise, the value of @var{parameter} is substituted. +@example +$ var= +$ : $@{var:?var is unset or null@} +bash: var: var is unset or null +@end example + @item $@{@var{parameter}:+@var{word}@} If @var{parameter} is null or unset, nothing is substituted, otherwise the expansion of @var{word} is substituted. +@example +$ var=123 +$ echo $@{var:+var is set and not null@} +var is set and not null +@end example + @item $@{@var{parameter}:@var{offset}@} @itemx $@{@var{parameter}:@var{offset}:@var{length}@} This is referred to as Substring Expansion. @@ -2389,21 +2414,78 @@ the pattern removal operation is applied to each member of the array in turn, and the expansion is the resultant list. @item $@{@var{parameter}/@var{pattern}/@var{string}@} - +@itemx $@{@var{parameter}//@var{pattern}/@var{string}@} +@itemx $@{@var{parameter}/#@var{pattern}/@var{string}@} +@itemx $@{@var{parameter}/%@var{pattern}/@var{string}@} The @var{pattern} is expanded to produce a pattern just as in filename expansion. @var{Parameter} is expanded and the longest match of @var{pattern} against its value is replaced with @var{string}. +@var{string} undergoes tilde expansion, parameter and variable expansion, +arithmetic expansion, command and process substitution, and quote removal. The match is performed according to the rules described below (@pxref{Pattern Matching}). -If @var{pattern} begins with @samp{/}, all matches of @var{pattern} are -replaced with @var{string}. Normally only the first match is replaced. -If @var{pattern} begins with @samp{#}, it must match at the beginning -of the expanded value of @var{parameter}. -If @var{pattern} begins with @samp{%}, it must match at the end -of the expanded value of @var{parameter}. -If @var{string} is null, matches of @var{pattern} are deleted + +In the first form above, only the first match is replaced. +If there are two slashes separating @var{parameter} and @var{pattern} +(the second form above), all matches of @var{pattern} are +replaced with @var{string}. +If @var{pattern} is preceded by @samp{#} (the third form above), +it must match at the beginning of the expanded value of @var{parameter}. +If @var{pattern} is preceded by @samp{%} (the fourth form above), +it must match at the end of the expanded value of @var{parameter}. +If the expansion of @var{string} is null, +matches of @var{pattern} are deleted. +If @var{string} is null, +matches of @var{pattern} are deleted and the @samp{/} following @var{pattern} may be omitted. + +Any unquoted instances of @samp{&} in @var{string} are replaced with the +matching portion of @var{pattern}. +This is intended to duplicate a common @code{sed} idiom. +Backslash is used to quote @samp{&} in @var{string}; the backslash is removed +in order to permit a literal @samp{&} in the replacement string. +Pattern substitution performs the check for @samp{&} after expanding +@var{string}, +so users should take care to quote backslashes intended to escape +the @samp{&} and inhibit replacement so they survive any quote removal +performed by the expansion of @var{string}. +For instance, + +@example +var=abcdef +echo $@{var/abc/& @} +echo "$@{var/abc/& @}" +echo $@{var/abc/"& "@} +@end example + +@noindent +will display three lines of "abc def", while + +@example +var=abcdef +echo $@{var/abc/\& @} +echo "$@{var/abc/\& @}" +echo $@{var/abc/"\& "@} +@end example + +@noindent +will display two lines of "abc def" and a third line of "& def". +The first two are replaced because the backslash is removed by quote +removal performed during the expansion of @var{string} +(the expansion is performed in a +context that doesn't take any enclosing double quotes into account, as +with other word expansions). +In the third case, the double quotes affect the expansion +of @samp{\&}, and, because @samp{&} is not one of the characters for +which backslash is special in double quotes, +the backslash survives the expansion, inhibits the replacement, +but is removed because it is treated specially. +One could use @samp{\\&}, unquoted, as the replacement string to achive +the same effect. +It should rarely be necessary to enclose only @var{string} in double +quotes. + If the @code{nocasematch} shell option (see the description of @code{shopt} in @ref{The Shopt Builtin}) is enabled, the match is performed without regard to the case @@ -2426,6 +2508,7 @@ filename expansion. Each character in the expanded value of @var{parameter} is tested against @var{pattern}, and, if it matches the pattern, its case is converted. The pattern should not attempt to match more than one character. + The @samp{^} operator converts lowercase letters matching @var{pattern} to uppercase; the @samp{,} operator converts matching uppercase letters to lowercase. @@ -2434,6 +2517,7 @@ expanded value; the @samp{^} and @samp{,} expansions match and convert only the first character in the expanded value. If @var{pattern} is omitted, it is treated like a @samp{?}, which matches every character. + If @var{parameter} is @samp{@@} or @samp{*}, the case modification operation is applied to each positional parameter in turn, and the expansion is the resultant list. diff --git a/doc/builtins.1 b/doc/builtins.1 index 4413c76f..611de93f 100644 --- a/doc/builtins.1 +++ b/doc/builtins.1 @@ -7,14 +7,14 @@ .de FN \fI\|\\$1\|\fP .. -.TH BASH_BUILTINS 1 "2004 Apr 20" "GNU Bash 5.0" +.TH BASH_BUILTINS 1 "2021 September 30" "GNU Bash 5.1" .SH NAME -bash, :, ., [, alias, bg, bind, break, builtin, caller, -cd, command, compgen, complete, -compopt, continue, declare, dirs, disown, echo, enable, eval, exec, exit, +:, ., [, alias, bg, bind, break, builtin, caller, +cd, command, compgen, complete, compopt, +continue, declare, dirs, disown, echo, enable, eval, exec, exit, export, false, fc, fg, getopts, hash, help, history, jobs, kill, let, local, logout, mapfile, popd, printf, pushd, pwd, read, -readonly, return, set, +readarray, readonly, return, set, shift, shopt, source, suspend, test, times, trap, true, type, typeset, ulimit, umask, unalias, unset, wait \- bash built-in commands, see \fBbash\fR(1) .SH BASH BUILTIN COMMANDS diff --git a/doc/version.texi b/doc/version.texi index ae9d1f7e..5bcf3069 100644 --- a/doc/version.texi +++ b/doc/version.texi @@ -2,10 +2,10 @@ Copyright (C) 1988-2021 Free Software Foundation, Inc. @end ignore -@set LASTCHANGE Mon Aug 23 10:07:59 EDT 2021 +@set LASTCHANGE Fri Oct 1 15:32:14 EDT 2021 @set EDITION 5.1 @set VERSION 5.1 -@set UPDATED 23 August 2021 -@set UPDATED-MONTH August 2021 +@set UPDATED 1 October 2021 +@set UPDATED-MONTH October 2021 diff --git a/execute_cmd.c b/execute_cmd.c index 12ad25bd..48680f6d 100644 --- a/execute_cmd.c +++ b/execute_cmd.c @@ -3903,8 +3903,7 @@ execute_cond_node (cond) arg1 = nullstr; if (echo_command_at_execute) xtrace_print_cond_term (cond->type, invert, cond->op, arg1, (char *)NULL); - /* TAG:bash-5.2 */ -#if 0 +#if 0 /* TAG:bash-5.2 */ if (varop && shell_compatibility_level > 51) /* -v */ #else if (varop) @@ -5615,12 +5614,9 @@ execute_disk_command (words, redirects, command_line, pipe_in, pipe_out, if (command) { /* If we're optimizing out the fork (implicit `exec'), decrement the - shell level like `exec' would do. */ -#if 0 /* TAG: bash-5.2 psmith 10/11/2020 */ + shell level like `exec' would do. Don't do this if we are already + in a pipeline environment, assuming it's already been done. */ if (nofork && pipe_in == NO_PIPE && pipe_out == NO_PIPE && (subshell_environment & SUBSHELL_PIPE) == 0) -#else - if (nofork && pipe_in == NO_PIPE && pipe_out == NO_PIPE) -#endif adjust_shell_level (-1); maybe_make_export_env (); diff --git a/lib/readline/rltty.c b/lib/readline/rltty.c index 4a492001..dd109331 100644 --- a/lib/readline/rltty.c +++ b/lib/readline/rltty.c @@ -694,6 +694,8 @@ rl_deprep_terminal (void) fprintf (rl_outstream, BRACK_PASTE_FINI); if (_rl_eof_found && (RL_ISSTATE (RL_STATE_TIMEOUT) == 0)) fprintf (rl_outstream, "\n"); + else if (_rl_echoing_p == 0) + fprintf (rl_outstream, "\n"); } if (_rl_enable_keypad) diff --git a/shell.h b/shell.h index 9805f36c..1bdbe721 100644 --- a/shell.h +++ b/shell.h @@ -85,6 +85,7 @@ extern int EOF_Reached; #define MATCH_QUOTED 0x020 #define MATCH_ASSIGNRHS 0x040 #define MATCH_STARSUB 0x080 +#define MATCH_EXPREP 0x100 /* for pattern substitution, expand replacement */ /* Some needed external declarations. */ extern char **shell_environment; diff --git a/subst.c b/subst.c index af087a6d..b22ac1ea 100644 --- a/subst.c +++ b/subst.c @@ -7686,7 +7686,7 @@ verify_substring_values (v, value, substr, vtype, e1p, e2p) from end of positional parameters? */ #if 1 if ((vtype == VT_ARRAYVAR || vtype == VT_POSPARMS) && *e2p < 0) -#else /* TAG: bash-5.2 */ +#else /* XXX - postponed; this isn't really a valuable feature */ if (vtype == VT_ARRAYVAR && *e2p < 0) #endif { @@ -8164,11 +8164,7 @@ parameter_brace_transform (varname, value, ind, xform, rtype, quoted, pflags, fl this_command_name = oname; if (vtype == VT_VARIABLE) FREE (val); -#if 0 /* TAG: bash-5.2 Martin Schulte 10/2020 */ return (interactive_shell ? &expand_param_error : &expand_param_fatal); -#else - return &expand_param_error; -#endif } starsub = vtype & VT_STARSUB; @@ -8377,7 +8373,6 @@ parameter_brace_substring (varname, value, ind, substr, quoted, pflags, flags) /* */ /****************************************************************/ -#if 0 /* TAG: bash-5.2 */ static int shouldexp_replacement (s) char *s; @@ -8395,6 +8390,12 @@ shouldexp_replacement (s) sindex++; if (s[sindex] == 0) return 0; + /* We want to remove this backslash because we treat it as special + in this context. THIS ASSUMES THE STRING IS PROCESSED BY + strcreplace() OR EQUIVALENT that handles removing backslashes + preceding the special character. */ + if (s[sindex] == '&') + return 1; } else if (c == '&') return 1; @@ -8402,7 +8403,6 @@ shouldexp_replacement (s) } return 0; } -#endif char * pat_subst (string, pat, rep, mflags) @@ -8418,12 +8418,7 @@ pat_subst (string, pat, rep, mflags) return (savestring ("")); mtype = mflags & MATCH_TYPEMASK; - -#if 0 /* TAG: bash-5.2 */ - rxpand = (rep && *rep) ? shouldexp_replacement (rep) : 0; -#else - rxpand = 0; -#endif + rxpand = mflags & MATCH_EXPREP; /* Special cases: * 1. A null pattern with mtype == MATCH_BEG means to prefix STRING @@ -8672,6 +8667,15 @@ parameter_brace_patsub (varname, value, ind, patsub, quoted, pflags, flags) rep = expand_string_if_necessary (rep, quoted, expand_string_unsplit); else rep = expand_string_to_string_internal (rep, quoted, expand_string_unsplit); + + /* Check whether or not to replace `&' in the replacement string after + expanding it, since we want to treat backslashes quoting the `&' + consistently. The replacement string already undergoes quote removal + above, so users need to make sure any desired backslash makes it + through that. */ + if (rep && *rep && shouldexp_replacement (rep)) + mflags |= MATCH_EXPREP; + } /* ksh93 doesn't allow the match specifier to be a part of the expanded diff --git a/tests/RUN-ONE-TEST b/tests/RUN-ONE-TEST index 0b063810..c8bef8dd 100755 --- a/tests/RUN-ONE-TEST +++ b/tests/RUN-ONE-TEST @@ -1,4 +1,4 @@ -BUILD_DIR=/usr/local/build/chet/bash/bash-current +BUILD_DIR=/usr/local/build/bash/bash-current THIS_SH=$BUILD_DIR/bash PATH=$PATH:$BUILD_DIR diff --git a/tests/mapfile.right b/tests/mapfile.right index 143d6ddc..4452232d 100644 --- a/tests/mapfile.right +++ b/tests/mapfile.right @@ -151,6 +151,7 @@ a [27] aaa [28] aaa [29] aaa +declare -a array=([0]="a" [1]="b" [2]="c" [3]=$'\n') 1 2 3 4 5 foo 0 1 diff --git a/tests/mapfile.tests b/tests/mapfile.tests index 01f1f2e5..c2ea7d24 100644 --- a/tests/mapfile.tests +++ b/tests/mapfile.tests @@ -52,6 +52,11 @@ for (( i = 0 ; i < ${#E[@]} ; i++ )); do echo "${E[${i}]}" done -${THIS_SH} ./mapfile1.sub +unset -v s array +s=$'a\xffb\xffc\xff' +mapfile -t -d $'\xff' array <<<"$s" +declare -p array +unset -v s array +${THIS_SH} ./mapfile1.sub ${THIS_SH} ./mapfile2.sub diff --git a/tests/new-exp.right b/tests/new-exp.right index 8a99fe98..2d243fe4 100644 --- a/tests/new-exp.right +++ b/tests/new-exp.right @@ -617,7 +617,7 @@ c Sub = 0 2 4 8 <> <> <'ab '\''cd'\'' ef'> -./new-exp10.sub: line 24: ${x@C}: bad substitution +bash: line 1: ${x@C}: bad substitution <'ab'> <'cd ef'> <''> <'gh'> <'ab' 'cd ef' '' 'gh'> <'ab'> <'cd> <''> <'gh'> @@ -729,6 +729,26 @@ a a a a +/homes/chetdefg +/homes/chetdefg +~defg +defg +defg +defg +$'a' $'b' $'c' $'d' $'e' $'f' $'g' +a b c d e f g +a b c d e f g +a b c d e f g +& & & & & & & +a a a a a a a +3 3 3 3 3 3 3 +abc defg +abc defg +abc defg +abc defg +abc defg +& defg +& defg argv[1] = argv[1] = diff --git a/tests/new-exp.tests b/tests/new-exp.tests index 079426cb..c542313f 100644 --- a/tests/new-exp.tests +++ b/tests/new-exp.tests @@ -631,6 +631,10 @@ ${THIS_SH} ./new-exp14.sub # ongoing work with a/A parameter transformations and `nounset' ${THIS_SH} ./new-exp15.sub +# pattern substitution with `&' (quoted and unquoted) in the replacement string +${THIS_SH} ./new-exp16.sub + + # problems with stray CTLNUL in bash-4.0-alpha unset a a=/a diff --git a/tests/new-exp10.sub b/tests/new-exp10.sub index bf9ebe7a..8c1f364e 100644 --- a/tests/new-exp10.sub +++ b/tests/new-exp10.sub @@ -21,7 +21,9 @@ printf "<%s>" "${x@A}" ; echo x="ab 'cd' ef" printf "<%s> " "${x@Q}" ; echo -printf "<%s>" "${x@C}" +# this needs to be run in a subshell because invalid transformation operators +# are now treated as substitution errors, fatal in non-interactive shells +${THIS_SH} -c 'x=abcdef ; printf "<%s>" "${x@C}"' bash # if unquoted, normal word splitting happens set -- ab 'cd ef' '' gh diff --git a/tests/new-exp16.sub b/tests/new-exp16.sub new file mode 100644 index 00000000..db89a317 --- /dev/null +++ b/tests/new-exp16.sub @@ -0,0 +1,31 @@ +HOME=/homes/chet +string=abcdefg +set -- a b c + +# verify existing behavior +echo ${string/abc/~} +echo "${string/abc/~}" +echo ${string/abc/"~"} + +echo ${string/abc/$notthere} +echo "${string/abc/$notthere}" +echo "${string/abc/"$notthere"}" + +echo ${string//?/\$\'&\' } + +echo ${string//?/\& } +echo "${string//?/\& }" +echo ${string//?/"& "} +echo ${string//?/"\& "} + +echo "${string//?/"a "}" +echo "${string//?/"$# "}" + +echo ${string/abc/& } +echo "${string/abc/& }" +echo ${string/abc/"& "} + +echo ${string/abc/\& } +echo "${string/abc/\& }" +echo ${string/abc/"\& "} +echo ${string/abc/\\& } diff --git a/trap.c b/trap.c index 702274a0..b4bb76b0 100644 --- a/trap.c +++ b/trap.c @@ -1455,7 +1455,7 @@ signal_in_progress (sig) return (sigmodes[sig] & SIG_INPROGRESS); } -#if 0 /* TAG: bash-5.2 */ +#if 0 /* unused */ int block_trapped_signals (maskp, omaskp) sigset_t *maskp;