commit bash-20200527 snapshot

lib/malloc/malloc.c

@@ -1,6 +1,6 @@
 /* malloc.c - dynamic memory allocation for bash. */
 
-/*  Copyright (C) 1985-2005 Free Software Foundation, Inc.
+/*  Copyright (C) 1985-2020 Free Software Foundation, Inc.
 
     This file is part of GNU Bash, the Bourne-Again SHell.
     
@@ -25,6 +25,8 @@
  *
  *	Nov 1983, Mike@BRL, Added support for 4.1C/4.2 BSD.
  *
+ * [VERY] old explanation:
+ *
  * This is a very fast storage allocator.  It allocates blocks of a small 
  * number of different sizes, and keeps free lists of each size.  Blocks
  * that don't exactly fit are passed up to the next larger size.  In this 
@@ -138,22 +140,35 @@
    enough room in the block for the new size.  Range checking is always
    done. */
 union mhead {
+#if SIZEOF_CHAR_P == 8
+  bits64_t mh_align[2];						/* 16 */
+#else
   bits64_t mh_align;						/* 8 */
+#endif
   struct {
     char mi_alloc; 		/* ISALLOC or ISFREE */		/* 1 */
     char mi_index;		/* index in nextf[] */		/* 1 */
     /* Remainder are valid only when block is allocated */
     u_bits16_t mi_magic2;	/* should be == MAGIC2 */	/* 2 */
     u_bits32_t mi_nbytes;	/* # of bytes allocated */	/* 4 */
+#if SIZEOF_CHAR_P == 8
+    char mi_magic8[8];		/* MAGIC1 guard bytes */	/* 8 */
+#endif
   } minfo;
 };
 #define mh_alloc	minfo.mi_alloc
 #define mh_index	minfo.mi_index
 #define mh_nbytes	minfo.mi_nbytes
 #define mh_magic2	minfo.mi_magic2
+#define mh_magic8	minfo.mi_magic8
 
 #define MOVERHEAD	sizeof(union mhead)
+
+#if SIZEOF_CHAR_P == 8
+#define MALIGN_MASK	15
+#else
 #define MALIGN_MASK	7	/* one less than desired alignment */
+#endif
 
 typedef union _malloc_guard {
   char s[4];
@@ -167,6 +182,8 @@ typedef union _malloc_guard {
    to describe the overhead for when the block is in use,
    and we do not want the free-list pointer to count in that.  */
 
+/* If SIZEOF_CHAR_P == 8, this goes into the mh_magic8 buffer at the end of
+   the rest of the struct. This may need adjusting. */
 #define CHAIN(a) \
   (*(union mhead **) (sizeof (char *) + (char *) (a)))
 
@@ -174,13 +191,14 @@ typedef union _malloc_guard {
    and end of each allocated block, and make sure they are undisturbed
    whenever a free or a realloc occurs. */
 
-/* Written in the 2 bytes before the block's real space (-4 bytes) */
+/* Written in the bytes before the block's real space (-SIZEOF_CHAR_P bytes) */
+#define MAGIC1 0x55
 #define MAGIC2 0x5555
 #define MSLOP  4		/* 4 bytes extra for u_bits32_t size */
 
 /* How many bytes are actually allocated for a request of size N --
-   rounded up to nearest multiple of 8 after accounting for malloc
-   overhead. */
+   rounded up to nearest multiple of 2*SIZEOF_CHAR_P after accounting for
+   malloc overhead. */
 #define ALLOCATED_BYTES(n) \
 	(((n) + MOVERHEAD + MSLOP + MALIGN_MASK) & ~MALIGN_MASK)
 
@@ -277,24 +295,24 @@ extern int errno;
 #endif
 
 /* Declarations for internal functions */
-static PTR_T internal_malloc __P((size_t, const char *, int, int));
-static PTR_T internal_realloc __P((PTR_T, size_t, const char *, int, int));
-static void internal_free __P((PTR_T, const char *, int, int));
-static PTR_T internal_memalign __P((size_t, size_t, const char *, int, int));
+static PTR_T internal_malloc PARAMS((size_t, const char *, int, int));
+static PTR_T internal_realloc PARAMS((PTR_T, size_t, const char *, int, int));
+static void internal_free PARAMS((PTR_T, const char *, int, int));
+static PTR_T internal_memalign PARAMS((size_t, size_t, const char *, int, int));
 #ifndef NO_CALLOC
-static PTR_T internal_calloc __P((size_t, size_t, const char *, int, int));
-static void internal_cfree __P((PTR_T, const char *, int, int));
+static PTR_T internal_calloc PARAMS((size_t, size_t, const char *, int, int));
+static void internal_cfree PARAMS((PTR_T, const char *, int, int));
 #endif
 #ifndef NO_VALLOC
-static PTR_T internal_valloc __P((size_t, const char *, int, int));
+static PTR_T internal_valloc PARAMS((size_t, const char *, int, int));
 #endif
 
 #if defined (botch)
 extern void botch ();
 #else
-static void botch __P((const char *, const char *, int));
+static void botch PARAMS((const char *, const char *, int));
 #endif
-static void xbotch __P((PTR_T, int, const char *, const char *, int));
+static void xbotch PARAMS((PTR_T, int, const char *, const char *, int));
 
 #if !HAVE_DECL_SBRK
 extern char *sbrk ();
@@ -302,7 +320,7 @@ extern char *sbrk ();
 
 #ifdef SHELL
 extern int interrupt_immediately, running_trap;
-extern int signal_is_trapped __P((int));
+extern int signal_is_trapped PARAMS((int));
 #endif
 
 #ifdef MALLOC_STATS
@@ -321,8 +339,8 @@ int malloc_mmap_threshold = MMAP_THRESHOLD;
 char _malloc_trace_buckets[NBUCKETS];
 
 /* These should really go into a header file. */
-extern void mtrace_alloc __P((const char *, PTR_T, size_t, const char *, int));
-extern void mtrace_free __P((PTR_T, int, const char *, int));
+extern void mtrace_alloc PARAMS((const char *, PTR_T, size_t, const char *, int));
+extern void mtrace_free PARAMS((PTR_T, int, const char *, int));
 #endif
 
 #if !defined (botch)
@@ -693,7 +711,7 @@ morecore (nu)
 
   memtop += sbrk_amt;
 
-  /* shouldn't happen, but just in case -- require 8-byte alignment */
+  /* shouldn't happen, but just in case -- require 8- or 16-byte alignment */
   if ((long)mp & MALIGN_MASK)
     {
       mp = (union mhead *) (((long)mp + MALIGN_MASK) & ~MALIGN_MASK);
@@ -723,8 +741,13 @@ malloc_debug_dummy ()
   write (1, "malloc_debug_dummy\n", 19);
 }
 
+#if SIZEOF_CHAR_P == 8
+#define PREPOP_BIN	3
+#define PREPOP_SIZE	64
+#else
 #define PREPOP_BIN	2
 #define PREPOP_SIZE	32
+#endif
 
 static int
 pagealign ()
@@ -760,8 +783,8 @@ pagealign ()
       memtop += sbrk_needed;
 
       /* Take the memory which would otherwise be wasted and populate the most
-	 popular bin (2 == 32 bytes) with it.  Add whatever we need to curbrk
-	 to make things 32-byte aligned, compute how many 32-byte chunks we're
+	 popular bin (3 == 64 bytes) with it.  Add whatever we need to curbrk
+	 to make things 64-byte aligned, compute how many 64-byte chunks we're
 	 going to get, and set up the bin. */
       curbrk += sbrk_needed & (PREPOP_SIZE - 1);
       sbrk_needed -= sbrk_needed & (PREPOP_SIZE - 1);
@@ -822,7 +845,7 @@ internal_malloc (n, file, line, flags)		/* get a block */
     if (nbytes <= binsize(nunits))
       break;
 
-  /* Silently reject too-large requests. */
+  /* Silently reject too-large requests. XXX - can increase this if HAVE_MMAP */
   if (nunits >= NBUCKETS)
     return ((PTR_T) NULL);
 
@@ -863,6 +886,11 @@ internal_malloc (n, file, line, flags)		/* get a block */
   p->mh_magic2 = MAGIC2;
   p->mh_nbytes = n;
 
+#if SIZEOF_CHAR_P == 8
+  /* Begin guard */
+  MALLOC_MEMSET ((char *)p->mh_magic8, MAGIC1, 8);
+#endif
+
   /* End guard */
   mg.i = n;
   z = mg.s;
@@ -897,6 +925,14 @@ internal_malloc (n, file, line, flags)		/* get a block */
     _malloc_ckwatch (p + 1, file, line, W_ALLOC, n);
 #endif
 
+#if defined (MALLOC_DEBUG)
+  z = (char *) (p + 1);
+  /* Check alignment of returned pointer */
+  if ((unsigned long)z & MALIGN_MASK)
+    fprintf (stderr, "malloc: %s:%d: warning: request for %ld bytes not aligned on %d byte boundary\r\n",
+	file ? file : _("unknown"), line, p->mh_nbytes, MALIGN_MASK+1);
+#endif
+
   return (PTR_T) (p + 1);
 }
 
@@ -956,6 +992,15 @@ internal_free (mem, file, line, flags)
   if (IN_BUCKET(nbytes, nunits) == 0)
     xbotch (mem, ERR_UNDERFLOW,
 	    _("free: underflow detected; mh_nbytes out of range"), file, line);
+#if SIZEOF_CHAR_P == 8
+  {
+    int i;
+    for (i = 0, z = p->mh_magic8; i < 8; i++)
+      if (*z++ != MAGIC1)
+	xbotch (mem, ERR_UNDERFLOW,
+		_("free: underflow detected; magic8 corrupted"), file, line);
+  }
+#endif
 
   ap += p->mh_nbytes;
   z = mg.s;
@@ -1087,6 +1132,16 @@ internal_realloc (mem, n, file, line, flags)
   if (IN_BUCKET(nbytes, nunits) == 0)
     xbotch (mem, ERR_UNDERFLOW,
 	    _("realloc: underflow detected; mh_nbytes out of range"), file, line);
+#if SIZEOF_CHAR_P == 8
+  {
+    int i;
+    for (i = 0, z = p->mh_magic8; i < 8; i++)
+      if (*z++ != MAGIC1)
+	xbotch (mem, ERR_UNDERFLOW,
+		_("realloc: underflow detected; magic8 corrupted"), file, line);
+
+  }
+#endif
 
   m = (char *)mem + (tocopy = p->mh_nbytes);
   z = mg.s;

lib/malloc/stats.c

@@ -110,7 +110,7 @@ _print_malloc_stats (s, fp)
       if (i == malloc_mmap_threshold+1)
 	fprintf (fp, "--------\n");
       if (v.nmal > 0)
-	fprintf (fp, "%8lu\t%4d\t%6d\t%5d\t%8d\t%8d %5d %8d\n", (unsigned long)v.blocksize, v.nfree, v.nused, v.nmal, v.nmorecore, v.nlesscore, v.nsplit, v.ncoalesce);
+	fprintf (fp, "%8lu\t%4d\t%6d\t%5d%8d\t%8d %5d %8d\n", (unsigned long)v.blocksize, v.nfree, v.nused, v.nmal, v.nmorecore, v.nlesscore, v.nsplit, v.ncoalesce);
       totfree += v.nfree * v.blocksize;
       totused += v.nused * v.blocksize;
     }

lib/readline/kill.c

@@ -820,7 +820,10 @@ _rl_bracketed_read_mbstring (char *mb, int mlen)
 #if defined (HANDLE_MULTIBYTE)
   if (MB_CUR_MAX > 1 && rl_byte_oriented == 0)
     c = _rl_read_mbstring (c, mb, mlen);
+  else
 #endif
+    mb[0] = c;
+  mb[mlen] = '\0';		/* just in case */
 
   return c;
 }

lib/readline/vi_mode.c

@@ -2014,10 +2014,11 @@ _rl_vi_callback_change_char (_rl_callback_generic_arg *data)
 
   c = _rl_vi_callback_getchar (mb, MB_LEN_MAX);
 #if defined (HANDLE_MULTIBYTE)
-  strncpy (_rl_vi_last_replacement, mb, MB_LEN_MAX);
-#else
-  _rl_vi_last_replacement[0] = c;
+  if (MB_CUR_MAX > 1 && rl_byte_oriented == 0)
+    strncpy (_rl_vi_last_replacement, mb, MB_LEN_MAX);
+  else
 #endif
+    _rl_vi_last_replacement[0] = c;
   _rl_vi_last_replacement[MB_LEN_MAX] = '\0';	/* XXX */
 
   if (c < 0)
@@ -2054,10 +2055,11 @@ rl_vi_change_char (int count, int key)
     {
       c = _rl_vi_callback_getchar (mb, MB_LEN_MAX);
 #ifdef HANDLE_MULTIBYTE
-      strncpy (_rl_vi_last_replacement, mb, MB_LEN_MAX);
-#else
-      _rl_vi_last_replacement[0] = c;
+      if (MB_CUR_MAX > 1 && rl_byte_oriented == 0)
+	strncpy (_rl_vi_last_replacement, mb, MB_LEN_MAX);
+      else
 #endif
+	_rl_vi_last_replacement[0] = c;
       _rl_vi_last_replacement[MB_LEN_MAX] = '\0';	/* just in case */      
     }
 

lib/sh/Makefile.in

@@ -2,7 +2,7 @@
 # Makefile for the Bash library
 #
 #
-# Copyright (C) 1998-2010 Free Software Foundation, Inc.
+# Copyright (C) 1998-2020 Free Software Foundation, Inc.
 
 #   This program is free software: you can redistribute it and/or modify
 #   it under the terms of the GNU General Public License as published by
@@ -93,7 +93,7 @@ CSOURCES = clktck.c clock.c getcwd.c getenv.c oslib.c setlinebuf.c \
 	   wcsdup.c fpurge.c zgetline.c mbscmp.c uconvert.c ufuncs.c \
 	   casemod.c dprintf.c input_avail.c mbscasecmp.c fnxform.c \
 	   strchrnul.c unicode.c wcswidth.c wcsnwidth.c shmbchar.c strdup.c \
-	   utf8.c
+	   utf8.c random.c
 
 # The header files for this library.
 HSOURCES = 
@@ -108,7 +108,7 @@ OBJECTS = clktck.o clock.o getenv.o oslib.o setlinebuf.o strnlen.o \
 	  fmtullong.o fmtumax.o zcatfd.o zmapfd.o winsize.o wcsdup.o \
 	  fpurge.o zgetline.o mbscmp.o uconvert.o ufuncs.o casemod.o \
 	  input_avail.o mbscasecmp.o fnxform.o unicode.o shmbchar.o \
-	  utf8.o wcsnwidth.o ${LIBOBJS}
+	  utf8.o random.o wcsnwidth.o ${LIBOBJS}
 
 SUPPORT = Makefile
 
@@ -170,6 +170,7 @@ netopen.o: netopen.c
 oslib.o: oslib.c
 pathcanon.o: pathcanon.c
 pathphys.o: pathphys.c
+random.o: random.c
 rename.o: rename.c
 setlinebuf.o: setlinebuf.c
 shmatch.o: shmatch.c
@@ -248,6 +249,7 @@ netopen.o: ${BUILD_DIR}/config.h
 oslib.o: ${BUILD_DIR}/config.h
 pathcanon.o: ${BUILD_DIR}/config.h
 pathphys.o: ${BUILD_DIR}/config.h
+random.o: ${BUILD_DIR}/config.h
 rename.o: ${BUILD_DIR}/config.h
 setlinebuf.o: ${BUILD_DIR}/config.h
 shmatch.o: ${BUILD_DIR}/config.h
@@ -380,6 +382,10 @@ pathphys.o: ${BASHINCDIR}/posixstat.h ${BASHINCDIR}/filecntl.h
 pathphys.o: ${BASHINCDIR}/ansi_stdlib.h ${BASHINCDIR}/chartypes.h
 #pathphys.o: ${BUILD_DIR}/version.h
 
+random.o: ${topdir}/bashtypes.h ${BASHINCDIR}/stdc.h
+random.o: ${topdir}/bashansi.h ${BASHINCDIR}/ansi_stdlib.h
+random.o: ${BASHINCDIR}/filecntl.h
+
 rename.o: ${topdir}/bashtypes.h ${BASHINCDIR}/stdc.h
 rename.o: ${BASHINCDIR}/posixstat.h
 

lib/sh/random.c

@@ -0,0 +1,240 @@
+/* random.c -- Functions for managing 16-bit and 32-bit random numbers. */
+
+/* Copyright (C) 2020 Free Software Foundation, Inc.
+
+   This file is part of GNU Bash, the Bourne Again SHell.
+
+   Bash is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   Bash is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with Bash.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "config.h"
+
+#include "bashtypes.h"
+
+#if defined (HAVE_SYS_RANDOM_H)
+#  include <sys/random.h>
+#endif
+
+#if defined (HAVE_UNISTD_H)
+#  include <unistd.h>
+#endif
+#include "filecntl.h"
+
+#include <stdio.h>
+#include "bashansi.h"
+
+#include "shell.h"
+
+extern time_t shell_start_time;
+
+extern int last_random_value;
+
+static u_bits32_t intrand32 PARAMS((u_bits32_t));
+static u_bits32_t genseed PARAMS((void));
+
+static u_bits32_t brand32 PARAMS((void));
+static void sbrand32 PARAMS((u_bits32_t));
+static void perturb_rand32 PARAMS((void));
+
+/* The random number seed.  You can change this by setting RANDOM. */
+static u_bits32_t rseed = 1;
+
+/* Returns a 32-bit pseudo-random number. */
+static u_bits32_t
+intrand32 (last)
+     u_bits32_t last;
+{
+  /* Minimal Standard generator from
+     "Random number generators: good ones are hard to find",
+     Park and Miller, Communications of the ACM, vol. 31, no. 10,
+     October 1988, p. 1195. Filtered through FreeBSD.
+
+     x(n+1) = 16807 * x(n) mod (m).
+
+     We split up the calculations to avoid overflow.
+
+     h = last / q; l = x - h * q; t = a * l - h * r
+     m = 2147483647, a = 16807, q = 127773, r = 2836
+
+     There are lots of other combinations of constants to use; look at
+     https://www.gnu.org/software/gsl/manual/html_node/Other-random-number-generators.html#Other-random-number-generators */
+
+  bits32_t h, l, t;
+  u_bits32_t ret;
+
+  /* Can't seed with 0. */
+  ret = (last == 0) ? 123459876 : last;
+  h = ret / 127773;
+  l = ret - (127773 * h);
+  t = 16807 * l - 2836 * h;
+  ret = (t < 0) ? t + 0x7fffffff : t;
+
+  return (ret);
+}
+
+static u_bits32_t
+genseed ()
+{
+  struct timeval tv;
+  u_bits32_t iv;
+
+  gettimeofday (&tv, NULL);
+  iv = (u_bits32_t)seedrand;		/* let the compiler truncate */
+  iv = tv.tv_sec ^ tv.tv_usec ^ getpid () ^ getppid () ^ current_user.uid ^ iv;
+  return (iv);
+}
+
+#define BASH_RAND_MAX	32767		/* 0x7fff - 16 bits */
+
+/* Returns a pseudo-random number between 0 and 32767. */
+int
+brand ()
+{
+  unsigned int ret;
+
+  rseed = intrand32 (rseed);
+  if (shell_compatibility_level > 50)
+    ret = (rseed >> 16) ^ (rseed & 65535);
+  else
+    ret = rseed;
+  return (ret & BASH_RAND_MAX);
+}
+
+/* Set the random number generator seed to SEED. */
+void
+sbrand (seed)
+     unsigned long seed;
+{
+  rseed = seed;
+  last_random_value = 0;
+}
+
+void
+seedrand ()
+{
+  u_bits32_t iv;
+
+  iv = genseed ();
+  sbrand (iv);
+}
+
+static u_bits32_t rseed32 = 1073741823;
+static int last_rand32;
+
+static int urandfd = -1;
+
+#define BASH_RAND32_MAX	0x7fffffff	/* 32 bits */
+
+/* Returns a 32-bit pseudo-random number between 0 and 4294967295. */
+static u_bits32_t
+brand32 ()
+{
+  u_bits32_t ret;
+
+  rseed32 = intrand32 (rseed32);
+  return (rseed32 & BASH_RAND32_MAX);
+}
+
+static void
+sbrand32 (seed)
+     u_bits32_t seed;
+{
+  last_rand32 = rseed32 = seed;
+}
+
+void
+seedrand32 ()
+{
+  u_bits32_t iv;
+
+  iv = genseed ();
+  sbrand32 (iv);
+}
+
+static void
+perturb_rand32 ()
+{
+  rseed32 ^= genseed ();
+}
+
+/* Force another attempt to open /dev/urandom on the next call to get_urandom32 */
+void
+urandom_close ()
+{
+  if (urandfd >= 0)
+    close (urandfd);
+  urandfd = -1;
+}
+
+#if !defined (HAVE_GETRANDOM)
+/* Imperfect emulation of getrandom(2). */
+#ifndef GRND_NONBLOCK
+#  define GRND_NONBLOCK 1
+#  define GRND_RANDOM 2
+#endif
+
+static ssize_t
+getrandom (buf, len, flags)
+     void *buf;
+     size_t len;
+     unsigned int flags;
+{
+  int oflags;
+  ssize_t r;
+  static int urand_unavail = 0;
+
+#if HAVE_GETENTROPY
+  r = getentropy (buf, len);
+  return (r == 0) ? len : -1;
+#endif
+
+  if (urandfd == -1 && urand_unavail == 0)
+    {
+      oflags = O_RDONLY;
+      if (flags & GRND_NONBLOCK)
+	oflags |= O_NONBLOCK;
+      urandfd = open ("/dev/urandom", oflags, 0);
+      if (urandfd >= 0)
+	SET_CLOSE_ON_EXEC (urandfd);
+      else
+	{
+	  urand_unavail = 1;
+	  return -1;
+	}
+    }
+  if (urandfd >= 0 && (r = read (urandfd, buf, len)) == len)
+    return (r);
+  return -1;
+}
+#endif
+      
+u_bits32_t
+get_urandom32 ()
+{
+  u_bits32_t ret;
+
+  if (getrandom ((void *)&ret, sizeof (ret), GRND_NONBLOCK) == sizeof (ret))
+    return (last_rand32 = ret);
+
+#if defined (HAVE_ARC4RANDOM)
+  ret = arc4random ();
+#else
+  if (subshell_environment)
+    perturb_rand32 ();
+  do
+    ret = brand32 ();
+  while (ret == last_rand32);
+#endif
+  return (last_rand32 = ret);
+}