fixes for array subscripts and values containing 0x01 characters

CWRU/CWRU.chlog

@@ -2105,3 +2105,25 @@ redir.c
 	- do_redirection_internal: if given [N]<&WORD- or [N]>&WORD- and WORD
 	  expands to null, make it identical to <&- or >&- and close file
 	  descriptor N (default 0). From a discussion back in 5/2021
+
+				   9/27
+				   ----
+arrayfunc.c
+	- expand_compound_array_assignment: since we run the string through
+	  the parser to turn it into a list (so we can make sure all shell
+	  metacharacters are properly quoted), we need to remove the CTLESC
+	  the parser uses to quote CTLESC and CTLNUL in *unquoted* words.
+	  The rest of the code assumes this has been done, and assumes that
+	  any CTLESC characters passed to expansion are part of the original
+	  word and should themselves be quoted, doubling the number of CTLESCs
+
+				   9/28
+				   ----
+arrayfunc.c
+	- expand_and_quote_kvpair_word,quote_compound_array_word,
+	  expand_and_quote_assoc_word,quote_compound_array_list: if we are
+	  single-quoting associative array subscripts and associative and
+	  indexed array values, we need to quote CTLESC characters, because
+	  that's how they come out of the parser and how the assignment
+	  statement code expects to see them.
+	  Fixes https://savannah.gnu.org/support/index.php?110538

MANIFEST

@@ -952,6 +952,7 @@ tests/assoc11.sub	f
 tests/assoc12.sub	f
 tests/assoc13.sub	f
 tests/assoc14.sub	f
+tests/assoc15.sub	f
 tests/attr.tests	f
 tests/attr.right	f
 tests/attr1.sub		f

arrayfunc.c

@@ -530,8 +530,17 @@ expand_compound_array_assignment (var, value, flags)
      shell expansions including pathname generation and word splitting. */
   /* First we split the string on whitespace, using the shell parser
      (ksh93 seems to do this). */
+  /* XXX - this needs a rethink, maybe use split_at_delims */
   list = parse_string_to_word_list (val, 1, "array assign");
 
+  /* If the parser has quoted CTLESC and CTNLNUL with CTLESC in unquoted
+     words, we need to remove those here because the code below assumes
+     they are there because they exist in the original word. */
+  /* XXX - if we rethink parse_string_to_word_list above, change this. */
+  for (nlist = list; nlist; nlist = nlist->next)
+    if ((nlist->word->flags & W_QUOTED) == 0)
+      remove_quoted_escapes (nlist->word->word);
+
   /* Note that we defer expansion of the assignment statements for associative
      arrays here, so we don't have to scan the subscript and find the ending
      bracket twice. See the caller below. */
@@ -616,15 +625,10 @@ expand_and_quote_kvpair_word (w)
   char *r, *s, *t;
 
   t = w ? expand_subscript_string (w, 0) : 0;
-#if 0	/* TAG:bash-5.2 */
   s = (t && strchr (t, CTLESC)) ? quote_escapes (t) : t;
   r = sh_single_quote (s ? s : "");
   if (s != t)
     free (s);
-#else
-  r = sh_single_quote (t ? t : "");
-#endif
-
   free (t);
   return r;
 }
@@ -907,7 +911,10 @@ quote_compound_array_word (w, type)
 
   wlen = strlen (w);
   w[ind] = '\0';
-  sub = sh_single_quote (w+1);
+  t = (strchr (w+1, CTLESC)) ? quote_escapes (w+1) : w+1;
+  sub = sh_single_quote (t);
+  if (t != w+1)
+   free (t);
   w[ind] = RBRACK;
 
   nword = xmalloc (wlen * 4 + 5);	/* wlen*4 is max single quoted length */
@@ -920,14 +927,10 @@ quote_compound_array_word (w, type)
   if (w[ind] == '+')
     nword[i++] = w[ind++];
   nword[i++] = w[ind++];
-#if 0	/* TAG:bash-5.2 */
   t = (strchr (w+ind, CTLESC)) ? quote_escapes (w+ind) : w+ind;
   value = sh_single_quote (t);
   if (t != w+ind)
    free (t);
-#else
-  value = sh_single_quote (w + ind);
-#endif
   strcpy (nword + i, value);
 
   return nword;
@@ -956,8 +959,11 @@ expand_and_quote_assoc_word (w, type)
 
   w[ind] = '\0';
   t = expand_subscript_string (w+1, 0);
+  s = (t && strchr (t, CTLESC)) ? quote_escapes (t) : t;
+  key = sh_single_quote (s ? s : "");
+  if (s != t)
+    free (s);
   w[ind] = RBRACK;
-  key = sh_single_quote (t ? t : "");
   free (t);
 
   wlen = STRLEN (key);
@@ -972,14 +978,10 @@ expand_and_quote_assoc_word (w, type)
   nword[i++] = w[ind++];
 
   t = expand_subscript_string (w+ind, 0);
-#if 0	/* TAG:bash-5.2 */
   s = (t && strchr (t, CTLESC)) ? quote_escapes (t) : t;
   value = sh_single_quote (s ? s : "");
   if (s != t)
     free (s);
-#else
-  value = sh_single_quote (t ? t : "");
-#endif
   free (t);
   nword = xrealloc (nword, wlen + 5 + STRLEN (value));
   strcpy (nword + i, value);
@@ -1008,14 +1010,10 @@ quote_compound_array_list (list, type)
 	continue;	/* should not happen, but just in case... */
       if ((l->word->flags & W_ASSIGNMENT) == 0)
 	{
-#if 0	/* TAG:bash-5.2 */
 	  s = (strchr (l->word->word, CTLESC)) ? quote_escapes (l->word->word) : l->word->word;
 	  t = sh_single_quote (s);
 	  if (s != l->word->word)
 	    free (s);
-#else
-	  t = sh_single_quote (l->word->word);
-#endif
 	}
       else 
 	t = quote_compound_array_word (l->word->word, type);

tests/array.right

@@ -760,8 +760,12 @@ declare -a bug3=([0]="" [1]="5" [2]="" [3]="1" [4]="")
 declare -a not_bug=([0]="no" [1]="nulls")
 declare -a workaround=([0]="")
 declare -a var=([0]=$'\001\001\001\001')
+declare -A v2=([$'\001']=$'ab\001c' )
 declare -a foo=([0]=$'\001\001\001\001')
-declare -a foo=([0]=$'\001\001')
-declare -a foo=([0]=$'\001\001')
-declare -A foo=([v]=$'\001\001' )
-declare -A foo=([v]=$'\001\001' )
+declare -A foo=([$'\001']=$'ab\001c' )
+declare -a foo=([0]=$'\001\001\001\001')
+declare -a foo=([0]=$'\001\001\001\001')
+declare -A foo=([v]=$'\001\001\001\001' )
+declare -A foo=([v]=$'\001\001\001\001' )
+declare -A foo=([$'\001']=$'ab\001c' )
+declare -A foo=([$'\001']=$'ab\001c' )

tests/array29.sub

@@ -11,8 +11,16 @@
 #   You should have received a copy of the GNU General Public License
 #   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
+# Issues with CTLESC characters in array subscripts and values. Bash-5.1 and
+# earlier didn't quote them correctly and therefore halved the number of
+# CTLESCs.
+
+declare -a var
 var=( $'\x01\x01\x01\x01' )
 declare -p var
+declare -A v2
+v2=( $'\x01' ab$'\x01'c )
+declare -p v2
 
 pv()
 {
@@ -22,6 +30,15 @@ pv()
 }
 pv
 
+unset -f pv
+pv()
+{
+        local -A foo
+        eval foo=\( "${v2[@]@k}" \)
+        declare -p foo
+}
+pv
+
 # these are wrong through bash-5.1; there is a fix tagged for bash-5.2
 # when I uncomment that fix, these results will reflect it
 
@@ -52,3 +69,18 @@ pv4()
 	declare -p foo
 }
 pv4
+
+unset -f pv3 pv4
+pv3()
+{
+	local -A foo=( $'\x01' "${v2[@]}" )
+	declare -p foo
+}
+pv3
+
+pv4()
+{
+	local -A foo=( [$'\x01']="${v2[@]}" )
+	declare -p foo
+}
+pv4

tests/assoc.right

@@ -329,3 +329,26 @@ argv[7] = <'foo'>
 argv[8] = <'bar'>
 declare -A clone=([hello]="world" ["key with spaces"]="value with spaces" [foo]="bar" [one]="1" )
 declare -A posparams=([hello]="world" ["key with spaces"]="value with spaces" [foo]="bar" [one]="1" )
+declare -A var=([$'\001']=$'\001\001\001\001' )
+declare -A v2=([$'\001']=$'\001\001\001\001' )
+argv[1] = <^A>
+argv[2] = <^A^A^A^A>
+declare -A foo=([$'\001']=$'\001\001\001\001' )
+declare -A var=([$'\001']=$'\001\001\001\001' )
+argv[1] = <^A>
+argv[2] = <^A^A^A^A>
+declare -A foo=([$'\001']=$'\001\001\001\001' )
+declare -A var=([$'\001']=$'\001\001\001\001' )
+argv[1] = <^A>
+argv[2] = <^A^A^A^A>
+declare -A foo=([$'\001']=$'\001\001\001\001' )
+declare -a var=([0]=$'\001\001\001\001')
+argv[1] = <$'\001\001\001\001'>
+declare -a foo=([0]=$'\001\001\001\001')
+declare -a var=([0]=$'\001\001\001\001')
+argv[1] = <$'\001\001\001\001'>
+declare -a foo=([0]=$'\001\001\001\001')
+declare -A var=([two]=$'ab\001cd' [one]=$'\001\001\001\001' )
+declare -A foo=([two]=$'ab\001cd' [one]=$'\001\001\001\001' )
+declare -A foo=([$'\001']=$'ab\001cd' )
+declare -A foo=([$'\001']=$'\001\001\001\001' )

tests/assoc.tests

@@ -253,3 +253,6 @@ ${THIS_SH} ./assoc13.sub
 
 # tests of the @k transformation on associative arrays
 ${THIS_SH} ./assoc14.sub
+
+# tests with subscripts and values containing 0x01 (some indexed array tests too)
+${THIS_SH} ./assoc15.sub

tests/assoc15.sub

@@ -0,0 +1,92 @@
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# associative arrays first
+
+v=$'\x01'
+
+declare -A var foo v2
+var=( $'\x01' $'\x01\x01\x01\x01' )
+declare -p var
+v2=( $v $v$v$v$v )
+declare -p v2
+
+recho "${var[@]@k}"
+eval foo=\( "${var[@]@k}" \)
+declare -p foo
+
+var=( 
 



 )
+declare -p var
+
+recho "${var[@]@k}"
+eval foo=\( "${var[@]@k}" \)
+declare -p foo
+
+var=( [
]=



 )
+declare -p var
+
+recho "${var[@]@k}"
+eval foo=\( "${var[@]@k}" \)
+declare -p foo
+
+# now indexed arrays
+unset -v var foo
+
+declare -a var
+var=( [0]=



 )
+declare -p var
+
+declare -a foo
+recho "${var[@]@Q}"
+eval foo=\( "${var[@]@Q}" \)
+declare -p foo
+
+var=( 



 )
+declare -p var
+
+unset foo
+
+declare -a foo
+recho "${var[@]@Q}"
+eval foo=\( "${var[@]@Q}" \)
+declare -p foo
+
+# similar to array29.sub
+unset -v var foo v2
+
+declare -A var
+var=( one $'\x01\x01\x01\x01' two ab$'\001'cd )
+declare -p var
+
+pv()
+{
+	local -A foo
+	eval foo=\( "${var[@]@k}" \)
+	declare -p foo
+}
+pv
+
+pv1()
+{
+	local -A foo=( $'\x01' "${var[two]}" )
+	declare -p foo
+}
+pv1
+
+pv2()
+{
+	local -A foo=( [$'\x01']="${var[one]}" )
+	declare -p foo
+}
+pv2