history library can now read history from non-regular files; fix for readline char search and macros; better fix for PROMPT_COMMAND and aliases ending in newlines; fix for casting COMMAND * and SIMPLE_COM * when parsing |&; fix to avoid undefined behavior when performing left and right arithmetic shifts

CWRU/CWRU.chlog

@@ -11417,3 +11417,67 @@ builtins/getopt.c,builtins/getopt.h
 builtins/getopts.def
 	- dogetopts: call sh_getopt_reset when binding the name variable fails
 	  for some reason
+
+				   7/18
+				   ----
+lib/readline/text.c
+	- _rl_char_search: make sure that character search arguments are
+	  added to any macro currently being defined
+	  Report from A4-Tacks <wdsjxhno1001@163.com>
+
+lib/readline/histfile.c
+	- read_history_slow: new function that just reads the history file in
+	  4096-byte chunks until EOF, used when the history file isn't a
+	  regular file.
+	- read_history_range: use read_history_slow when the history file isn't
+	  a regular file
+	  Suggested by several, most recently by <macbeth.112358@gmail.com> in 2/25
+
+				   7/19
+				   ----
+parse.y,input.h,eval.c,array.h
+	- revert changes from 7/6 based on report from
+	  Sam James <sam@gentoo.org>
+
+parse.y
+	- exec_restore_parser_state: version of restore_parser_state that
+	  doesn't restore the pushed string list, since some calls to
+	  parse_and_execute may modify it out from underneath us
+	- parse_comsub: use exec_restore_parser_state instead of inline code
+	  (the version in parse_compound_assignment is a little more complex)
+	- parser_unset_string_list: extern function to set pushed_string_list
+	  to NULL from outside parse.y
+	- execute_variable_command: set pushed_string_list to NULL after
+	  saving the parser state, since we don't want parse_and_execute to
+	  pop a string off the list that was there before it was called
+	  Fixes bug reported by Carl Johnson <carl.johnson.new.hampshire@gmail.com>
+
+shell.h
+	- exec_restore_parser_state: extern declaration
+	- parser_unset_string_list: extern declaration
+
+trap.c
+	- _run_trap_internal: use parser_unset_string_list after saving
+	  parser_state to fix bug with DEBUG trap if it's invoked during
+	  an alias ending with a newline (similar to issue from 7/6)
+	- run_pending_traps: call parser_unset_string_list as above
+
+				   7/21
+				   ----
+bashline.c
+	- bash_execute_unix_command,edit_and_execute_command: use
+	  parser_unset_string_list after saving parser_state; executing a
+	  command while readline is active shouldn't modify any existing
+	  parser state
+
+				   7/25
+				   ----
+command.h
+	- revert changes to COMMAND and SIMPLE_COM from 7/16
+	  From a report by Jessica Clarke <jrtc27@jrtc27.com>
+
+expr.c
+	- expassign,expshift: avoid C23 undefined behavior when performing
+	  left and right arithmetic shifts
+	  From https://savannah.gnu.org/patch/?10532
+	  bkallus <benjamin.p.kallus.gr@dartmouth.edu>

array.h

@@ -177,6 +177,6 @@ extern arrayind_t element_back (ARRAY *, arrayind_t);
 #define ALL_ELEMENT_SUB(c)	((c) == '@' || (c) == '*')
 
 /* In eval.c, but uses ARRAY * */
-extern int execute_array_command (ARRAY *, void *, int);
+extern int execute_array_command (ARRAY *, void *);
 
 #endif /* _ARRAY_H_ */

bashline.c

@@ -984,6 +984,7 @@ edit_and_execute_command (int count, int c, int editing_mode, const char *edit_c
     (*rl_deprep_term_function) ();
   rl_clear_signals ();
   save_parser_state (&ps);
+  parser_unset_string_list ();
   r = parse_and_execute (command, (editing_mode == VI_EDITING_MODE) ? "v" : "C-xC-e", SEVAL_NOHIST);
   restore_parser_state (&ps);
 
@@ -4680,6 +4681,7 @@ bash_execute_unix_command (int count, int key)
 
   begin_unwind_frame ("execute-unix-command");
   save_parser_state (&ps);
+  parser_unset_string_list ();
   rl_clear_signals ();
   add_unwind_protect (uw_unbind_readline_variables, 0);
   add_unwind_protect (uw_restore_parser_state, &ps);

command.h

@@ -195,6 +195,7 @@ typedef struct element {
 
 /* What a command looks like. */
 typedef struct command {
+  enum command_type type;	/* FOR CASE WHILE IF CONNECTION or SIMPLE. */
   int flags;			/* Flags controlling execution environment. */
   int line;			/* line number the command starts on */
   REDIRECT *redirects;		/* Special redirects for FOR CASE, etc. */
@@ -222,7 +223,6 @@ typedef struct command {
     struct subshell_com *Subshell;
     struct coproc_com *Coproc;
   } value;
-  enum command_type type;	/* FOR CASE WHILE IF CONNECTION SIMPLE, etc. */
 } COMMAND;
 
 /* Structure used to represent the CONNECTION type. */
@@ -337,9 +337,9 @@ typedef struct cond_com {
 typedef struct simple_com {
   int flags;			/* See description of CMD flags. */
   int line;			/* line number the command starts on */
-  REDIRECT *redirects;		/* Redirections to perform. */
   WORD_LIST *words;		/* The program name, the arguments,
 				   variable assignments, etc. */
+  REDIRECT *redirects;		/* Redirections to perform. */
 } SIMPLE_COM;
 
 /* The "function definition" command. */

configure

@@ -12749,7 +12749,8 @@ else case e in #(
            # defined sleep(n) _sleep ((n) * 1000)
            #endif
            #include <fcntl.h>
-           GL_MDA_DEFINES
+
+
            #ifndef O_NOATIME
             #define O_NOATIME 0
            #endif

eval.c

@@ -283,7 +283,7 @@ send_pwd_to_eterm (void)
 #if defined (ARRAY_VARS)
 /* Caller ensures that A has a non-zero number of elements */
 int
-execute_array_command (ARRAY *a, void *v, int flags)
+execute_array_command (ARRAY *a, void *v)
 {
   char *tag;
   char **argv;
@@ -295,7 +295,7 @@ execute_array_command (ARRAY *a, void *v, int flags)
   for (i = 0; i < argc; i++)
     {
       if (argv[i] && argv[i][0])
-	execute_variable_command (argv[i], tag, flags);
+	execute_variable_command (argv[i], tag);
     }
   strvec_dispose (argv);
   return 0;
@@ -318,7 +318,7 @@ execute_prompt_command (void)
   if (array_p (pcv))
     {
       if ((pcmds = array_cell (pcv)) && array_num_elements (pcmds) > 0)
-	execute_array_command (pcmds, "PROMPT_COMMAND", 0);
+	execute_array_command (pcmds, "PROMPT_COMMAND");
       return;
     }
   else if (assoc_p (pcv))
@@ -327,7 +327,7 @@ execute_prompt_command (void)
 
   command_to_execute = value_cell (pcv);
   if (command_to_execute && *command_to_execute)
-    execute_variable_command (command_to_execute, "PROMPT_COMMAND", 0);
+    execute_variable_command (command_to_execute, "PROMPT_COMMAND");
 }
 
 /* Call the YACC-generated parser and return the status of the parse.

examples/loadables/strptime.c

@@ -188,7 +188,8 @@ strptime_builtin (WORD_LIST *list)
 {
   char *s;
   struct tm t, *tm;
-  time_t now, secs;
+  time_t now;
+  intmax_t secs;
   char *datestr, *format;
   int i, opt;
 
@@ -227,7 +228,7 @@ strptime_builtin (WORD_LIST *list)
       if (STREQ (datestr, date_time_modifiers[i].shorthand))
 	{
 	  secs = now + date_time_modifiers[i].incr;
-	  printf ("%ld\n", secs);    
+	  printf ("%jd\n", secs);    
 	  return (EXECUTION_SUCCESS);
 	}
     }
@@ -265,7 +266,7 @@ strptime_builtin (WORD_LIST *list)
   if (s && *s)
     builtin_warning("%s: not completely converted (%s)", datestr, s);
 
-  printf ("%ld\n", secs);    
+  printf ("%jd\n", secs);    
   return (EXECUTION_SUCCESS);
 }
 

expr.c

@@ -145,10 +145,6 @@
    lowest precedence. */
 #define EXP_LOWEST	expcomma
 
-#ifndef MAX_INT_LEN
-#  define MAX_INT_LEN 32
-#endif
-
 struct lvalue
 {
   char *tokstr;		/* possibly-rewritten lvalue if not NULL */
@@ -581,10 +577,10 @@ expassign (void)
 	      lvalue -= value;
 	      break;
 	    case LSH:
-	      lvalue <<= value;
+	      lvalue = (uintmax_t)lvalue << (value & (TYPE_WIDTH(uintmax_t) - 1));
 	      break;
 	    case RSH:
-	      lvalue >>= value;
+	      lvalue >>= (value & (TYPE_WIDTH(uintmax_t) - 1));
 	      break;
 	    case BAND:
 	      lvalue &= value;
@@ -841,7 +837,7 @@ expcompare (void)
 static intmax_t
 expshift (void)
 {
-  register intmax_t val1, val2;
+  intmax_t val1, val2;
 
   val1 = expaddsub ();
 
@@ -853,9 +849,9 @@ expshift (void)
       val2 = expaddsub ();
 
       if (op == LSH)
-	val1 = val1 << val2;
+	val1 = (uintmax_t)val1 << (val2 & (TYPE_WIDTH(uintmax_t) - 1));
       else
-	val1 = val1 >> val2;
+	val1 = val1 >> (val2 & (TYPE_WIDTH(uintmax_t) - 1));
       lasttok = NUM;
     }
 

input.h

@@ -99,7 +99,7 @@ extern int stream_on_stack (enum stream_type);
 extern char *read_secondary_line (int);
 extern int find_reserved_word (const char *);
 extern void gather_here_documents (void);
-extern void execute_variable_command (const char *, const char *, int);
+extern void execute_variable_command (const char *, const char *);
 
 extern int *save_token_state (void);
 extern void restore_token_state (int *);

lib/readline/histfile.c

@@ -58,6 +58,7 @@
 #  include <unistd.h>
 #endif
 
+#include <string.h>
 #include <ctype.h>
 
 #if defined (__EMX__)
@@ -145,6 +146,9 @@ static int histfile_backup (const char *, const char *);
 static int histfile_restore (const char *, const char *);
 static int history_rename (const char *, const char *);
 
+static int history_write_slow (int, HIST_ENTRY **, int, int);
+static ssize_t history_read_slow (int, char **);
+
 /* Return the string that should be used in the place of this
    filename.  This only matters when you don't specify the
    filename to read_history (), or write_history (). */
@@ -258,6 +262,69 @@ read_history (const char *filename)
   return (read_history_range (filename, 0, -1));
 }
 
+#define RBUFSIZE	4096
+
+/* Read from a non-regular file until EOF, assuming we can't trust the file
+   size as reported by fstat. */
+static ssize_t
+history_read_slow (int fd, char **bufp)
+{
+  char *ret, *r;
+  size_t retsize, retlen;
+  char rbuf[RBUFSIZE];
+  ssize_t nr, nw;
+
+  if (bufp == 0)
+    return -1;
+
+  retsize = RBUFSIZE;
+  ret = malloc(retsize);
+  if (ret == 0)
+    return -1;
+  retlen = 0;
+
+  while (nr = read (fd, rbuf, sizeof (rbuf)))
+    {
+      if (nr < 0)
+	{
+	  free (ret);
+	  *bufp = NULL;
+	  return -1;
+	}
+
+      if (retlen >= retsize - nr - 1)
+	{
+	  retsize *= 2;
+	  r = realloc (ret, retsize);
+	  if (r == 0)
+	    {
+	      free(ret);
+	      *bufp = NULL;
+	      return -1;
+	    }
+	  ret = r;
+	}
+      memcpy (ret + retlen, rbuf, nr);
+      retlen += nr;
+    }
+  if (retlen + 1 >= retsize)
+    {
+      retsize += 1;
+      r = realloc (ret, retsize);
+      if (r == 0)
+	{
+	  free (ret);
+	  *bufp = NULL;
+	  return -1;
+	}
+      ret = r;
+    }		
+  ret[retlen] = '\0';
+
+  *bufp = ret;
+  return (ssize_t)retlen;
+}
+
 /* Read a range of lines from FILENAME, adding them to the history list.
    Start reading at the FROM'th line and end at the TO'th.  If FROM
    is zero, start at the beginning.  If TO is less than FROM, read
@@ -294,12 +361,15 @@ read_history_range (const char *filename, int from, int to)
 
   if (S_ISREG (finfo.st_mode) == 0)
     {
-#ifdef EFTYPE
-      errno = EFTYPE;
-#else
-      errno = EINVAL;
-#endif
-      goto error_and_exit;
+      chars_read = history_read_slow (file, &buffer);
+      if (chars_read == 0)
+	{
+	  free (buffer);
+	  free (input);
+	  close (file);
+	  return 0;
+	}
+      goto after_file_read;
     }
   else
     {
@@ -341,6 +411,8 @@ read_history_range (const char *filename, int from, int to)
 
   chars_read = read (file, buffer, file_size);
 #endif
+
+after_file_read:
   if (chars_read < 0)
     {
   error_and_exit:

lib/readline/misc.c

@@ -137,6 +137,7 @@ _rl_arg_dispatch (_rl_arg_cxt cxt, int c)
       else
 	{
 	  key = _rl_bracketed_read_key ();
+	  /* XXX - add to macro def? */
 	  rl_restore_prompt ();
 	  rl_clear_message ();
 	  RL_UNSETSTATE(RL_STATE_NUMERICARG);

lib/readline/text.c

@@ -1786,13 +1786,17 @@ static int
 _rl_char_search (int count, int fdir, int bdir)
 {
   char mbchar[MB_LEN_MAX];
-  int mb_len;
+  int mb_len, i;
 
   mb_len = _rl_read_mbchar (mbchar, MB_LEN_MAX);
 
   if (mb_len <= 0)
     return 1;
 
+  if (RL_ISSTATE (RL_STATE_MACRODEF))
+    for (i = 0; i < mb_len; i++)
+      _rl_add_macro_char (mbchar[i]);
+
   if (count < 0)
     return (_rl_char_search_internal (-count, bdir, mbchar, mb_len));
   else
@@ -1805,9 +1809,13 @@ _rl_char_search (int count, int fdir, int bdir)
   int c;
 
   c = _rl_bracketed_read_key ();
+
   if (c < 0)
     return 1;
 
+  if (RL_ISSTATE (RL_STATE_MACRODEF))
+    _rl_add_macro_char (c);
+
   if (count < 0)
     return (_rl_char_search_internal (-count, bdir, c));
   else

m4/fcntl-o.m4

@@ -32,7 +32,8 @@ AC_DEFUN([gl_FCNTL_O_FLAGS],
            # defined sleep(n) _sleep ((n) * 1000)
            #endif
            #include <fcntl.h>
-           ]GL_MDA_DEFINES[
+           ]
+	   [
            #ifndef O_NOATIME
             #define O_NOATIME 0
            #endif

parse.y

@@ -1473,23 +1473,22 @@ pipeline:	pipeline '|' newline_list pipeline
 	|	pipeline BAR_AND newline_list pipeline
 			{
 			  /* Make cmd1 |& cmd2 equivalent to cmd1 2>&1 | cmd2 */
-			  COMMAND *tc;
 			  REDIRECTEE rd, sd;
-			  REDIRECT *r;
+			  REDIRECT *r, **rp;
 
-			  tc = $1->type == cm_simple ? (COMMAND *)$1->value.Simple : $1;
+			  rp = $1->type == cm_simple ? &$1->value.Simple->redirects : &$1->redirects;
 			  sd.dest = 2;
 			  rd.dest = 1;
 			  r = make_redirection (sd, r_duplicating_output, rd, 0);
-			  if (tc->redirects)
+			  if (*rp)
 			    {
 			      register REDIRECT *t;
-			      for (t = tc->redirects; t->next; t = t->next)
+			      for (t = *rp; t->next; t = t->next)
 				;
 			      t->next = r;
 			    }
 			  else
-			    tc->redirects = r;
+			    *rp = r;
 
 			  $$ = command_connect ($1, $4, '|');
 			}
@@ -2166,6 +2165,12 @@ parser_restore_alias (void)
 #endif
 }
 
+void
+parser_unset_string_list (void)
+{
+  pushed_string_list = (STRING_SAVER *)NULL;
+}
+
 #if defined (ALIAS)
 /* Before freeing AP, make sure that there aren't any cases of pointer
    aliasing that could cause us to reference freed memory later on. */
@@ -3008,19 +3013,18 @@ discard_until (int character)
 }
 
 void
-execute_variable_command (const char *command, const char *vname, int flags)
+execute_variable_command (const char *command, const char *vname)
 {
   char *last_lastarg;
   sh_parser_state_t ps;
 
-  if (flags)
-    save_parser_state (&ps);
+  save_parser_state (&ps);
+  pushed_string_list = (STRING_SAVER *)NULL;
   last_lastarg = save_lastarg ();
 
   parse_and_execute (savestring (command), vname, SEVAL_NONINT|SEVAL_NOHIST|SEVAL_NOOPTIMIZE|SEVAL_NOTIFY);
 
-  if (flags)
-    restore_parser_state (&ps);
+  restore_parser_state (&ps);
   bind_lastarg (last_lastarg);
   FREE (last_lastarg);
 
@@ -4459,7 +4463,6 @@ parse_comsub (int qc, int open, int close, size_t *lenp, int flags)
   char *ret, *tcmd;
   size_t retlen;
   sh_parser_state_t ps;
-  STRING_SAVER *saved_strings;
   COMMAND *saved_global, *parsed_command;
 
   /* Posix interp 217 says arithmetic expressions have precedence, so
@@ -4625,9 +4628,7 @@ INTERNAL_DEBUG(("current_token (%d) != shell_eof_token (%c)", current_token, she
   /* We don't want to restore the old pushed string list, since we might have
      used it to consume additional input from an alias while parsing this
      command substitution. */
-  saved_strings = pushed_string_list;
-  restore_parser_state (&ps);
-  pushed_string_list = saved_strings;
+  exec_restore_parser_state (&ps);
 
   simplecmd_lineno = save_lineno;
 
@@ -7355,6 +7356,20 @@ uw_restore_parser_state (void *ps)
   restore_parser_state (ps);
 }
 
+/* Special version of restore parser state for cases where we called
+   parse_and_execute(), which may have modified the pushed string list
+   out from underneath us. We may need to use this in other places,
+   like running traps while processing aliases. */
+void
+exec_restore_parser_state (sh_parser_state_t *ps)
+{
+  STRING_SAVER *ss;
+
+  ss = pushed_string_list;
+  restore_parser_state (ps);
+  pushed_string_list = ss;
+}
+
 /* Free the parts of a parser state struct that have allocated memory. */
 void
 flush_parser_state (sh_parser_state_t *ps)

shell.h

@@ -247,6 +247,8 @@ extern sh_parser_state_t *save_parser_state (sh_parser_state_t *);
 extern void restore_parser_state (sh_parser_state_t *);
 extern void flush_parser_state (sh_parser_state_t *);
 extern void uw_restore_parser_state (void *);
+extern void exec_restore_parser_state (sh_parser_state_t *);
+extern void parser_unset_string_list (void);
 
 extern sh_input_line_state_t *save_input_line_state (sh_input_line_state_t *);
 extern void restore_input_line_state (sh_input_line_state_t *);

tests/new-exp2.sub

@@ -17,7 +17,7 @@ export LANG=C
 # test out the new $(< filename) code
 # it should be exactly equivalent to $(cat filename)
 
-FILENAME=$TMPDIR/bashtmp.x$$
+FILENAME=$TMPDIR/bashtmp.x$$ ; rm -f $TMPDIR/bashtmp.x*
 
 trap 'rm -f $FILENAME' 0
 

tests/redir.right

@@ -156,6 +156,8 @@ bix ()
     echo foo 2>&1 | cat
 }
 foo
+foo
+foo
 ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 1
 7

tests/redir9.sub

@@ -61,3 +61,7 @@ echo foo |& cat
 type bix
 
 bix
+
+echo foo |& tee $TMPDIR/bar
+cat $TMPDIR/bar
+rm -f $TMPDIR/bar

trap.c

@@ -467,6 +467,7 @@ run_pending_traps (void)
 	      trap_command = savestring (old_trap);
 
 	      save_parser_state (&pstate);
+	      parser_unset_string_list ();
 	      save_subst_varlist = subst_assign_varlist;
 	      subst_assign_varlist = 0;
 	      save_tempenv = temporary_env;
@@ -1160,6 +1161,8 @@ _run_trap_internal (int sig, char *tag)
 #endif
 
       save_parser_state (&pstate);
+      parser_unset_string_list ();
+
       save_subst_varlist = subst_assign_varlist;
       subst_assign_varlist = 0;
       save_tempenv = temporary_env;