optimize many cases of $(<file) not to require a fork

CWRU/CWRU.chlog

@@ -1784,3 +1784,52 @@ lib/readline/kill.c
 	  or more slashes. The old code went too far and deleted the previous
 	  word as well. From dabe@dabe.com
 
+				   8/31
+				   ----
+parse.y
+	- STRING_SAVER: now save and restore shell_input_line_len; not sure
+	  why it wasn't done before; fix push_string and pop_string accordingly
+	- prompt_again: now takes a parameter FORCE; not used yet (every
+	  caller passes 0), needs more thought
+
+builtins/evalstring.h
+	- open_redir_file: broke code that expands the redirection and opens
+	  the resultant filename into a new function, called from cat_file
+
+redir.h
+	- open_redir_file: extern declaration here for now
+
+builtins/evalstring.c
+	- parse_string: takes a new argument: COMMAND **cmdp; if non-null, saves
+	  the parsed command to *cmdp and lets the caller manage it itself.
+	  global_command is still not modified. Changed callers in parse.y
+
+				    9/1
+				    ---
+parse.y
+	- parse_string_to_command: stripped-down version of xparse_dolparen
+	  that takes a string, runs it through the parser, and returns the
+	  resultant COMMAND *; uses parse_string with the new argument
+
+externs.h
+	- parse_string_to_command: extern declaration
+
+builtins/evalstring.c
+	- can_optimize_cat_file: new function, takes a COMMAND * argument and
+	  returns true if the command can be optimized like $(<file); changed
+	  parse_and_execute to call it
+
+subst.c
+	- optimize_cat_file: new function, optimizes $(<file) without creating
+	  a new process. Uses redir_open to open the redirection file, after
+	  expansion, and calls read_comsub to read from it directly
+	- read_comsub: now reads into a 4096 byte buffer (COMSUB_PIPEBUF)
+	- command_substitute: if the string begins with a `<' and isn't followed
+	  by any of "<>&", see if we can optimize the command and call
+	  optimize_cat_file to do it if we can. 
+
+
+
+
+
+

Makefile.in

@@ -427,11 +427,12 @@ BASHINCFILES =	 $(BASHINCDIR)/posixstat.h $(BASHINCDIR)/ansi_stdlib.h \
 		 $(BASHINCDIR)/shtty.h $(BASHINCDIR)/typemax.h \
 		 $(BASHINCDIR)/ocache.h
 
-LIBRARIES = $(GLOB_LIB) $(SHLIB_LIB) $(READLINE_LIB) $(HISTORY_LIB) $(TERMCAP_LIB) \
-	    $(TILDE_LIB) $(MALLOC_LIB) $(INTL_LIB) $(LIBICONV) $(LOCAL_LIBS)
+LIBRARIES = $(GLOB_LIB) $(SHLIB_LIB) $(READLINE_LIB) $(HISTORY_LIB) \
+	    $(TERMCAP_LIB) $(TILDE_LIB) $(MALLOC_LIB) $(INTL_LIB) $(LIBICONV) \
+	    $(LOCAL_LIBS)
 
-LIBDEP = $(GLOB_DEP) $(SHLIB_DEP) $(INTL_DEP) $(READLINE_DEP) $(HISTORY_DEP) $(TERMCAP_DEP) \
-	 $(TILDE_DEP) $(MALLOC_DEP)
+LIBDEP = $(GLOB_DEP) $(SHLIB_DEP) $(INTL_DEP) $(READLINE_DEP) $(HISTORY_DEP) \
+	 $(TERMCAP_DEP) $(TILDE_DEP) $(MALLOC_DEP)
 
 LIBRARY_LDFLAGS = $(READLINE_LDFLAGS) $(HISTORY_LDFLAGS) $(GLOB_LDFLAGS) \
 		  $(TILDE_LDFLAGS) $(MALLOC_LDFLAGS) $(SHLIB_LDFLAGS)
@@ -741,7 +742,7 @@ syntax.c:	mksyntax${EXEEXT} $(srcdir)/syntax.h
 	$(RM) $@
 	./mksyntax$(EXEEXT) -o $@
 
-$(BUILTINS_LIBRARY): $(BUILTIN_DEFS) $(BUILTIN_C_SRC) config.h ${BASHINCDIR}/memalloc.h $(DEFDIR)/builtext.h version.h
+$(BUILTINS_LIBRARY): $(BUILTIN_DEFS) $(BUILTIN_C_SRC) config.h ${BASHINCDIR}/memalloc.h  version.h $(DEFDIR)/builtext.h
 	@(cd $(DEFDIR) && $(MAKE) $(MFLAGS) DEBUG=${DEBUG} targets ) || exit 1
 
 # these require special rules to circumvent make builtin rules
@@ -1455,7 +1456,7 @@ builtins/evalstring.o: ${BASHINCDIR}/memalloc.h variables.h arrayfunc.h conftype
 builtins/evalstring.o: quit.h unwind_prot.h ${BASHINCDIR}/maxpath.h jobs.h builtins.h
 builtins/evalstring.o: dispose_cmd.h make_cmd.h subst.h externs.h 
 builtins/evalstring.o: jobs.h builtins.h flags.h input.h execute_cmd.h
-builtins/evalstring.o: bashhist.h $(DEFSRC)/common.h pathnames.h
+builtins/evalstring.o: bashhist.h $(DEFSRC)/common.h pathnames.h redir.h
 builtins/evalstring.o: ${DEFDIR}/builtext.h
 builtins/getopt.o: config.h ${BASHINCDIR}/memalloc.h
 builtins/getopt.o: shell.h syntax.h bashjmp.h command.h general.h xmalloc.h error.h

builtins/common.h

@@ -40,7 +40,7 @@ do { \
     builtin_help (); \
     return (EX_USAGE)
 
-/* Flag values for parse_and_execute () */
+/* Flag values for parse_and_execute () and parse_string () */
 #define SEVAL_NONINT	0x001
 #define SEVAL_INTERACT	0x002
 #define SEVAL_NOHIST	0x004
@@ -213,9 +213,10 @@ extern WORD_LIST *get_directory_stack PARAMS((int));
 extern int parse_and_execute PARAMS((char *, const char *, int));
 extern int evalstring PARAMS((char *, const char *, int));
 extern void parse_and_execute_cleanup PARAMS((int));
-extern int parse_string PARAMS((char *, const char *, int, char **));
+extern int parse_string PARAMS((char *, const char *, int, COMMAND **, char **));
 extern int should_suppress_fork PARAMS((COMMAND *));
 extern int can_optimize_connection PARAMS((COMMAND *));
+extern int can_optimize_cat_file PARAMS((COMMAND *));
 extern void optimize_fork PARAMS((COMMAND *));
 extern void optimize_subshell_command PARAMS((COMMAND *));
 extern void optimize_shell_function PARAMS((COMMAND *));

builtins/evalstring.c

@@ -173,6 +173,19 @@ optimize_shell_function (command)
     }  
 }
 
+int
+can_optimize_cat_file (command)
+     COMMAND *command;
+{
+  return (command->type == cm_simple && !command->redirects &&
+	    (command->flags & CMD_TIME_PIPELINE) == 0 &&
+	    command->value.Simple->words == 0 &&
+	    command->value.Simple->redirects &&
+	    command->value.Simple->redirects->next == 0 &&
+	    command->value.Simple->redirects->instruction == r_input_direction &&
+	    command->value.Simple->redirects->redirector.dest == 0);
+}
+
 /* How to force parse_and_execute () to clean up after itself. */
 void
 parse_and_execute_cleanup (old_running_trap)
@@ -471,13 +484,7 @@ parse_and_execute (string, from_file, flags)
 	      if (startup_state == 2 &&
 		  (subshell_environment & SUBSHELL_COMSUB) &&
 		  *bash_input.location.string == '\0' &&
-		  command->type == cm_simple && !command->redirects &&
-		  (command->flags & CMD_TIME_PIPELINE) == 0 &&
-		  command->value.Simple->words == 0 &&
-		  command->value.Simple->redirects &&
-		  command->value.Simple->redirects->next == 0 &&
-		  command->value.Simple->redirects->instruction == r_input_direction &&
-		  command->value.Simple->redirects->redirector.dest == 0)
+		  can_optimize_cat_file (command))
 		{
 		  int r;
 		  r = cat_file (command->value.Simple->redirects);
@@ -541,10 +548,11 @@ parse_and_execute (string, from_file, flags)
    command substitutions during parsing to obey Posix rules about finding
    the end of the command and balancing parens. */
 int
-parse_string (string, from_file, flags, endp)
+parse_string (string, from_file, flags, cmdp, endp)
      char *string;
      const char *from_file;
      int flags;
+     COMMAND **cmdp;
      char **endp;
 {
   int code, nc;
@@ -619,7 +627,10 @@ itrace("parse_string: longjmp executed: code = %d", code);
 	  
       if (parse_command () == 0)
 	{
-	  dispose_command (global_command);
+	  if (cmdp)
+	    *cmdp = global_command;
+	  else
+	    dispose_command (global_command);
 	  global_command = (COMMAND *)NULL;
 	}
       else
@@ -665,12 +676,10 @@ out:
   return (nc);
 }
 
-/* Handle a $( < file ) command substitution.  This expands the filename,
-   returning errors as appropriate, then just cats the file to the standard
-   output. */
-static int
-cat_file (r)
+int
+open_redir_file (r, fnp)
      REDIRECT *r;
+     char **fnp;
 {
   char *fn;
   int fd, rval;
@@ -696,9 +705,30 @@ cat_file (r)
     {
       file_error (fn);
       free (fn);
+      if (fnp)
+	*fnp = 0;
       return -1;
     }
 
+  if (fnp)
+    *fnp = fn;
+  return fd;
+}
+
+/* Handle a $( < file ) command substitution.  This expands the filename,
+   returning errors as appropriate, then just cats the file to the standard
+   output. */
+static int
+cat_file (r)
+     REDIRECT *r;
+{
+  char *fn;
+  int fd, rval;
+
+  fd = open_redir_file (r, &fn);
+  if (fd < 0)
+    return -1;
+
   rval = zcatfd (fd, 1, fn);
 
   free (fn);

externs.h

@@ -109,6 +109,7 @@ extern int yyparse PARAMS((void));
 extern int return_EOF PARAMS((void));
 extern void push_token PARAMS((int));
 extern char *xparse_dolparen PARAMS((char *, char *, int *, int));
+extern COMMAND *parse_string_to_command PARAMS((char *, int));
 extern void reset_parser PARAMS((void));
 extern void reset_readahead_token PARAMS((void));
 extern WORD_LIST *parse_string_to_word_list PARAMS((char *, int, const char *));

parse.y

@@ -200,7 +200,7 @@ static void print_offending_line PARAMS((void));
 static void report_syntax_error PARAMS((char *));
 
 static void handle_eof_input_unit PARAMS((void));
-static void prompt_again PARAMS((void));
+static void prompt_again PARAMS((int));
 #if 0
 static void reset_readline_prompt PARAMS((void));
 #endif
@@ -1873,7 +1873,7 @@ typedef struct string_saver {
 #if defined (ALIAS)
   alias_t *expander;   /* alias that caused this line to be pushed. */
 #endif
-  size_t saved_line_size, saved_line_index;
+  size_t saved_line_size, saved_line_index, saved_line_len;
   int saved_line_terminator;
   int flags;
 } STRING_SAVER;
@@ -1899,6 +1899,7 @@ push_string (s, expand, ap)
   temp->expand_alias = expand;
   temp->saved_line = shell_input_line;
   temp->saved_line_size = shell_input_line_size;
+  temp->saved_line_len = shell_input_line_len;
   temp->saved_line_index = shell_input_line_index;
   temp->saved_line_terminator = shell_input_line_terminator;
   temp->flags = 0;
@@ -1916,7 +1917,7 @@ push_string (s, expand, ap)
 #endif
 
   shell_input_line = s;
-  shell_input_line_size = STRLEN (s);
+  shell_input_line_size = shell_input_line_len = STRLEN (s);
   shell_input_line_index = 0;
   shell_input_line_terminator = '\0';
 #if 0
@@ -1941,6 +1942,7 @@ pop_string ()
   shell_input_line = pushed_string_list->saved_line;
   shell_input_line_index = pushed_string_list->saved_line_index;
   shell_input_line_size = pushed_string_list->saved_line_size;
+  shell_input_line_len = pushed_string_list->saved_line_len;
   shell_input_line_terminator = pushed_string_list->saved_line_terminator;
 
   if (pushed_string_list->expand_alias)
@@ -2152,8 +2154,8 @@ read_secondary_line (remove_quoted_newline)
   int n, c;
 
   prompt_string_pointer = &ps2_prompt;
-  if (SHOULD_PROMPT())
-    prompt_again ();
+  if (SHOULD_PROMPT ())
+    prompt_again (0);
   ret = read_a_line (remove_quoted_newline);
 #if defined (HISTORY)
   if (ret && remember_on_history && (parser_state & PST_HEREDOC))
@@ -2554,7 +2556,7 @@ shell_getc (remove_quoted_newline)
 	  shell_input_line_size = 0;
 	  prompt_string_pointer = &current_prompt_string;
 	  if (SHOULD_PROMPT ())
-	    prompt_again ();
+	    prompt_again (0);
 	  goto restart_read;
 	}
 
@@ -2662,7 +2664,7 @@ pop_alias:
   if MBTEST(uc == '\\' && remove_quoted_newline && shell_input_line[shell_input_line_index] == '\n')
     {
 	if (SHOULD_PROMPT ())
-	  prompt_again ();
+	  prompt_again (0);
 	line_number++;
 
 	/* What do we do here if we're expanding an alias whose definition
@@ -2886,7 +2888,7 @@ yylex ()
       /* Avoid printing a prompt if we're not going to read anything, e.g.
 	 after resetting the parser with read_token (RESET). */
       if (token_to_read == 0 && SHOULD_PROMPT ())
-	prompt_again ();
+	prompt_again (0);
     }
 
   two_tokens_ago = token_before_that;
@@ -3698,7 +3700,7 @@ parse_matched_pair (qc, open, close, lenp, flags)
 
       /* Possible reprompting. */
       if (ch == '\n' && SHOULD_PROMPT ())
-	prompt_again ();
+	prompt_again (0);
 
       /* Don't bother counting parens or doing anything else if in a comment
 	 or part of a case statement */
@@ -4190,7 +4192,7 @@ xparse_dolparen (base, string, indp, flags)
 
   token_to_read = DOLPAREN;			/* let's trick the parser */
 
-  nc = parse_string (string, "command substitution", sflags, &ep);
+  nc = parse_string (string, "command substitution", sflags, (COMMAND **)NULL, &ep);
 
   /* Should we save and restore the bison/yacc lookahead token (yychar) here?
      Or only if it's not YYEMPTY? */
@@ -4264,6 +4266,72 @@ xparse_dolparen (base, string, indp, flags)
   return ret;
 }
 
+/* Recursively call the parser to parse the string from a $(...) command
+   substitution to a COMMAND *. This is called from command_substitute() and
+   has the same parser state constraints as xparse_dolparen(). */
+COMMAND *
+parse_string_to_command (string, flags)
+     char *string;
+     int flags;
+{
+  sh_parser_state_t ps;
+  sh_input_line_state_t ls;
+  int nc, sflags;
+  size_t slen;
+  char *ret, *ep;
+  COMMAND *cmd;
+
+  if (*string == 0)
+    return (COMMAND *)NULL;
+
+  ep = string;
+  slen = STRLEN (string);
+
+/*itrace("parse_string_to_command: size = %d shell_input_line = `%s' string=`%s'", shell_input_line_size, shell_input_line, string);*/
+
+  sflags = SEVAL_NONINT|SEVAL_NOHIST|SEVAL_NOFREE;
+  if (flags & SX_NOLONGJMP)
+    sflags |= SEVAL_NOLONGJMP;
+
+  save_parser_state (&ps);
+  save_input_line_state (&ls);
+
+#if defined (ALIAS) || defined (DPAREN_ARITHMETIC)
+  pushed_string_list = (STRING_SAVER *)NULL;
+#endif
+  if (flags & SX_COMPLETE)
+    parser_state |= PST_NOERROR;
+
+  cmd = 0;
+  nc = parse_string (string, "command substitution", sflags, &cmd, &ep);
+
+  reset_parser ();
+  /* reset_parser() clears shell_input_line and associated variables, including
+     parser_state, so we want to reset things, then restore what we need. */
+  restore_input_line_state (&ls);
+  restore_parser_state (&ps);
+
+  /* If parse_string returns < 0, we need to jump to top level with the
+     negative of the return value. We abandon the rest of this input line
+     first */
+  if (nc < 0)
+    {
+      clear_shell_input_line ();	/* XXX */
+      if ((flags & SX_NOLONGJMP) == 0)
+        jump_to_top_level (-nc);	/* XXX */
+    }
+
+  /* Need to check how many characters parse_string() consumed, make sure it's
+     the entire string. */
+  if (nc < slen)
+    {
+      dispose_command (cmd);
+      return (COMMAND *)NULL;
+    }
+
+  return cmd;
+}
+
 #if defined (DPAREN_ARITHMETIC) || defined (ARITH_FOR_COMMAND)
 /* Parse a double-paren construct.  It can be either an arithmetic
    command, an arithmetic `for' command, or a nested subshell.  Returns
@@ -4439,7 +4507,7 @@ cond_skip_newlines ()
   while ((cond_token = read_token (READ)) == '\n')
     {
       if (SHOULD_PROMPT ())
-	prompt_again ();
+	prompt_again (0);
     }
   return (cond_token);
 }
@@ -5023,7 +5091,7 @@ got_escaped_character:
 
     next_character:
       if (character == '\n' && SHOULD_PROMPT ())
-	prompt_again ();
+	prompt_again (0);
 
       /* We want to remove quoted newlines (that is, a \<newline> pair)
 	 unless we are within single quotes or pass_next_character is
@@ -5374,7 +5442,8 @@ history_delimiting_chars (line)
 /* Issue a prompt, or prepare to issue a prompt when the next character
    is read. */
 static void
-prompt_again ()
+prompt_again (force)
+     int force;
 {
   char *temp_prompt;
 
@@ -6150,7 +6219,7 @@ handle_eof_input_unit ()
 	      last_read_token = current_token = '\n';
 	      /* Reset the prompt string to be $PS1. */
 	      prompt_string_pointer = (char **)NULL;
-	      prompt_again ();
+	      prompt_again (0);
 	      return;
 	    }
 	}
@@ -6314,7 +6383,7 @@ parse_compound_assignment (retlenp)
       if (tok == '\n')			/* Allow newlines in compound assignments */
 	{
 	  if (SHOULD_PROMPT ())
-	    prompt_again ();
+	    prompt_again (0);
 	  continue;
 	}
       if (tok != WORD && tok != ASSIGNMENT_WORD)

redir.h

@@ -37,4 +37,7 @@ extern int do_redirections PARAMS((REDIRECT *, int));
 extern char *redirection_expand PARAMS((WORD_DESC *));
 extern int stdin_redirects PARAMS((REDIRECT *));
 
+/* in builtins/evalstring.c for now, could move later */
+extern int open_redir_file PARAMS((REDIRECT *, char **));
+
 #endif /* _REDIR_H_ */

subst.c

@@ -45,6 +45,7 @@
 
 #include "shell.h"
 #include "parser.h"
+#include "redir.h"
 #include "flags.h"
 #include "jobs.h"
 #include "execute_cmd.h"
@@ -306,6 +307,7 @@ static int valid_parameter_transform PARAMS((char *));
 
 static char *process_substitute PARAMS((char *, int));
 
+static char *optimize_cat_file PARAMS((REDIRECT *, int, int, int *));
 static char *read_comsub PARAMS((int, int, int, int *));
 
 #ifdef ARRAY_VARS
@@ -6281,12 +6283,32 @@ process_substitute (string, open_for_read_in_child)
 /*				   */
 /***********************************/
 
+#define COMSUB_PIPEBUF	4096
+
+static char *
+optimize_cat_file (r, quoted, flags, flagp)
+     REDIRECT *r;
+     int quoted, flags, *flagp;
+{
+  char *ret;
+  int fd;
+
+  fd = open_redir_file (r, (char **)0);
+  if (fd < 0)
+    return &expand_param_error;
+
+  ret = read_comsub (fd, quoted, flags, flagp);
+  close (fd);
+
+  return ret;
+}
+
 static char *
 read_comsub (fd, quoted, flags, rflag)
      int fd, quoted, flags;
      int *rflag;
 {
-  char *istring, buf[512], *bufp;
+  char *istring, buf[COMSUB_PIPEBUF], *bufp;
   int istring_index, c, tflag, skip_ctlesc, skip_ctlnul;
   int mb_cur_max;
   size_t istring_size;
@@ -6431,15 +6453,38 @@ command_substitute (string, quoted, flags)
 
   /* Don't fork () if there is no need to.  In the case of no command to
      run, just return NULL. */
-#if 1
   for (s = string; s && *s && (shellblank (*s) || *s == '\n'); s++)
     ;
   if (s == 0 || *s == 0)
     return ((WORD_DESC *)NULL);
-#else
-  if (!string || !*string || (string[0] == '\n' && !string[1]))
-    return ((WORD_DESC *)NULL);
-#endif
+
+  if (*s == '<' && (s[1] != '<' && s[1] != '>' && s[1] != '&'))
+    {
+      COMMAND *cmd;
+
+      cmd = parse_string_to_command (string, 0);	/* XXX - flags */
+      if (cmd && can_optimize_cat_file (cmd))
+	{
+	  tflag = 0;
+	  istring = optimize_cat_file (cmd->value.Simple->redirects, quoted, flags, &tflag);
+	  if (istring == &expand_param_error)
+	    {
+	      last_command_exit_value = EXECUTION_FAILURE;
+	      istring = 0;
+	    }
+	  else
+	    last_command_exit_value = EXECUTION_SUCCESS;	/* compat */
+	  last_command_subst_pid = dollar_dollar_pid;
+
+	  dispose_command (cmd);	  
+	  ret = alloc_word_desc ();
+	  ret->word = istring;
+	  ret->flags = tflag;
+
+	  return ret;
+	}
+      dispose_command (cmd);
+    }
 
   if (wordexp_only && read_but_dont_execute)
     {

tests/RUN-ONE-TEST

@@ -1,4 +1,4 @@
-BUILD_DIR=/usr/local/build/chet/bash/bash-current
+BUILD_DIR=/usr/local/build/bash/bash-current
 THIS_SH=$BUILD_DIR/bash
 PATH=$PATH:$BUILD_DIR