diff --git a/CWRU/CWRU.chlog b/CWRU/CWRU.chlog index 16835c92..782a778b 100644 --- a/CWRU/CWRU.chlog +++ b/CWRU/CWRU.chlog @@ -12127,7 +12127,7 @@ doc/{bash.1,bashref.texi} lib/glob/sm_loop.c - parse_collsym: make sure to not return an out-of-bounds read if a collating symbol is unterminated. Fixes OOB read reported by - op7ic \x00 + Jerzy Kramarz - brackmatch: after incrementing p, before checking whether it's a character range, check whether *p was NULL before the increment and short-circuit the bracket expression if it is @@ -12282,12 +12282,12 @@ subst.c - parameter_brace_{patsub,remove_pattern,transform,casemod}: save and restore this_command_name while temporarily setting it for use in error messages. Fixes use-after-free error reported by - op7ic \x00 + Jerzy Kramarz - string_extract_verbatim: make sure when we increment i by 2 due to a CTLESC or CTLESC-CTLNUL that we don't read past the end of the string. This can happen if the string ends with an odd number of CTLESC chars. Fixes oob-read error reported by - op7ic \x00 + Jerzy Kramarz 11/11 ----- @@ -12355,3 +12355,37 @@ lib/sh/unicode.c - u32cconv: even if we don't have iconv, if locale_utf8locale is non- zero, return u32toutf8 + 11/15 + ----- +lib/readline/nls.c + - _rl_init_locale: new function, split off code that determines current + locale from _rl_init_eightbit, use it to set _rl_utf8locale + +lib/readiline/rlprivate.h + - _rl_init_locale: new extern declaration + +lib/readline/readline.c + - rl_initialize: if not initializing everything the first time, call + _rl_init_locale to check current locale and set _rl_utf8locale + +lib/readline/text.c + - _rl_insert_char: optimize cases where we are inserting a single-byte + character in a locale with multibyte characters + - _rl_insert_char: check whether character is single byte if we know + we are in a UTF-8 locale, optimize single-byte case to avoid calls + to mbrtowc and memmove/memcpy + +lib/readline/mbutil.c + - _rl_char_value: if we are in a UTF-8 locale (_rl_utf8locale) and the + current character does not have the eighth bit set ((c & 0x80) == 0), + return that char without bothering to call mbrtowc + - _rl_adjust_point: don't bother calling mbrlen if we are in a UTF-8 + locale and the current character is not a multibyte character + - _rl_find_next_mbchar_internal: if we are in a UTF-8 locale, use that + to avoid calls to mbrtowc + +lib/readline/display.c + - _rl_col_width: if in a UTF-8 locale, take advantage of that to avoid + calls to mbrlen and mbrtowc + - rl_redisplay: if in a UTF-8 locale, take advantage of that to avoid + calls to mbrtowc diff --git a/RUN-ONE-TEST b/RUN-ONE-TEST new file mode 100755 index 00000000..d29259a9 --- /dev/null +++ b/RUN-ONE-TEST @@ -0,0 +1,10 @@ +BUILD_DIR=/usr/local/build/chet/bash/bash-current +THIS_SH=$BUILD_DIR/bash +PATH=$PATH:$BUILD_DIR + +export THIS_SH PATH + +export BASH_TSTOUT=/tmp/xx +rm -f ${BASH_TSTOUT} + +/bin/sh "$@" diff --git a/include/shmbutil.h b/include/shmbutil.h index dbaaba86..264cae33 100644 --- a/include/shmbutil.h +++ b/include/shmbutil.h @@ -37,6 +37,7 @@ extern size_t mbstrlen __P((const char *)); extern char *xstrchr __P((const char *, int)); extern int locale_mb_cur_max; /* XXX */ +extern int locale_utf8locale; /* XXX */ #ifndef MB_INVALIDCH #define MB_INVALIDCH(x) ((x) == (size_t)-1 || (x) == (size_t)-2) @@ -109,6 +110,8 @@ extern int locale_mb_cur_max; /* XXX */ _f = is_basic ((_str)[_i]); \ if (_f) \ mblength = 1; \ + else if (locale_utf8locale && (((_str)[_i] & 0x80) == 0)) \ + mblength = 1; \ else \ { \ state_bak = state; \ @@ -267,6 +270,8 @@ extern int locale_mb_cur_max; /* XXX */ _k = is_basic (*(_src)); \ if (_k) \ mblength = 1; \ + else if (locale_utf8locale && ((*(_src) & 0x80) == 0)) \ + mblength = 1; \ else \ { \ state_bak = state; \ diff --git a/lib/readline/display.c b/lib/readline/display.c index cd4f3fed..0e0d44c3 100644 --- a/lib/readline/display.c +++ b/lib/readline/display.c @@ -124,7 +124,6 @@ static int _rl_col_width PARAMS((const char *, int, int, int)); #define PROMPT_ENDING_INDEX \ ((MB_CUR_MAX > 1 && rl_byte_oriented == 0) ? prompt_physical_chars : prompt_last_invisible+1) - /* **************************************************************** */ /* */ /* Display stuff */ @@ -844,8 +843,13 @@ rl_redisplay () if (mb_cur_max > 1 && rl_byte_oriented == 0) { memset (&ps, 0, sizeof (mbstate_t)); - /* XXX - what if wc_bytes ends up <= 0? check for MB_INVALIDCH */ - wc_bytes = mbrtowc (&wc, rl_line_buffer, rl_end, &ps); + if (_rl_utf8locale && UTF8_SINGLEBYTE(rl_line_buffer[0])) + { + wc = (wchar_t)rl_line_buffer[0]; + wc_bytes = 1; + } + else + wc_bytes = mbrtowc (&wc, rl_line_buffer, rl_end, &ps); } else wc_bytes = 1; @@ -1005,13 +1009,18 @@ rl_redisplay () if (mb_cur_max > 1 && rl_byte_oriented == 0) { in += wc_bytes; - /* XXX - what if wc_bytes ends up <= 0? check for MB_INVALIDCH */ - wc_bytes = mbrtowc (&wc, rl_line_buffer + in, rl_end - in, &ps); + if (_rl_utf8locale && UTF8_SINGLEBYTE(rl_line_buffer[in])) + { + wc = (wchar_t)rl_line_buffer[in]; + wc_bytes = 1; + memset (&ps, 0, sizeof (mbstate_t)); /* re-init state */ + } + else + wc_bytes = mbrtowc (&wc, rl_line_buffer + in, rl_end - in, &ps); } else in++; #endif - } line[out] = '\0'; if (cpos_buffer_position < 0) @@ -2961,7 +2970,13 @@ _rl_col_width (str, start, end, flags) while (point < start) { - tmp = mbrlen (str + point, max, &ps); + if (_rl_utf8locale && UTF8_SINGLEBYTE(str[point])) + { + memset (&ps, 0, sizeof (mbstate_t)); + tmp = 1; + } + else + tmp = mbrlen (str + point, max, &ps); if (MB_INVALIDCH ((size_t)tmp)) { /* In this case, the bytes are invalid or too short to compose a @@ -2990,7 +3005,13 @@ _rl_col_width (str, start, end, flags) while (point < end) { - tmp = mbrtowc (&wc, str + point, max, &ps); + if (_rl_utf8locale && UTF8_SINGLEBYTE(str[point])) + { + tmp = 1; + wc = (wchar_t) str[point]; + } + else + tmp = mbrtowc (&wc, str + point, max, &ps); if (MB_INVALIDCH ((size_t)tmp)) { /* In this case, the bytes are invalid or too short to compose a diff --git a/lib/readline/mbutil.c b/lib/readline/mbutil.c index 314a49b6..df1c4843 100644 --- a/lib/readline/mbutil.c +++ b/lib/readline/mbutil.c @@ -105,7 +105,14 @@ _rl_find_next_mbchar_internal (string, seed, count, find_non_zero) len = strlen (string + point); if (len == 0) break; - tmp = mbrtowc (&wc, string+point, len, &ps); + if (_rl_utf8locale && UTF8_SINGLEBYTE(string[point])) + { + tmp = 1; + wc = (wchar_t) string[point]; + memset(&ps, 0, sizeof(mbstate_t)); + } + else + tmp = mbrtowc (&wc, string+point, len, &ps); if (MB_INVALIDCH ((size_t)tmp)) { /* invalid bytes. assume a byte represents a character */ @@ -169,7 +176,7 @@ _rl_find_prev_mbchar_internal (string, seed, find_non_zero) tmp = mbrtowc (&wc, string + point, length - point, &ps); if (MB_INVALIDCH ((size_t)tmp)) { - /* in this case, bytes are invalid or shorted to compose + /* in this case, bytes are invalid or too short to compose multibyte char, so assume that the first byte represents a single character anyway. */ tmp = 1; @@ -283,10 +290,13 @@ _rl_adjust_point (string, point, ps) while (pos < point) { - tmp = mbrlen (string + pos, length - pos, ps); + if (_rl_utf8locale && UTF8_SINGLEBYTE(string[pos])) + tmp = 1; + else + tmp = mbrlen (string + pos, length - pos, ps); if (MB_INVALIDCH ((size_t)tmp)) { - /* in this case, bytes are invalid or shorted to compose + /* in this case, bytes are invalid or too short to compose multibyte char, so assume that the first byte represents a single character anyway. */ pos++; @@ -334,6 +344,8 @@ _rl_char_value (buf, ind) if (MB_LEN_MAX == 1 || rl_byte_oriented) return ((wchar_t) buf[ind]); + if (_rl_utf8locale && UTF8_SINGLEBYTE(buf[ind])) + return ((wchar_t) buf[ind]); l = strlen (buf); if (ind >= l - 1) return ((wchar_t) buf[ind]); diff --git a/lib/readline/nls.c b/lib/readline/nls.c index d3b36344..7f3e2d8a 100644 --- a/lib/readline/nls.c +++ b/lib/readline/nls.c @@ -74,6 +74,7 @@ static char *legal_lang_values[] = "iso88599", "iso885910", "koi8r", + "utf8", 0 }; @@ -118,16 +119,12 @@ utf8locale (lspec) #endif } -/* Check for LC_ALL, LC_CTYPE, and LANG and use the first with a value - to decide the defaults for 8-bit character input and output. Returns - 1 if we set eight-bit mode. */ -int -_rl_init_eightbit () +/* Query the right environment variables and call setlocale() to initialize + the C library locale settings. */ +char * +_rl_init_locale () { -/* If we have setlocale(3), just check the current LC_CTYPE category - value, and go into eight-bit mode if it's not C or POSIX. */ -#if defined (HAVE_SETLOCALE) - char *lspec, *t; + char *ret, *lspec; /* Set the LC_CTYPE locale category from environment variables. */ lspec = _rl_get_locale_var ("LC_CTYPE"); @@ -140,10 +137,25 @@ _rl_init_eightbit () lspec = setlocale (LC_CTYPE, (char *)NULL); if (lspec == 0) lspec = ""; - t = setlocale (LC_CTYPE, lspec); + ret = setlocale (LC_CTYPE, lspec); /* ok, since it does not change locale */ - if (t && *t) - _rl_utf8locale = utf8locale (t); + _rl_utf8locale = (ret && *ret) ? utf8locale (ret) : 0; + + return ret; +} + +/* Check for LC_ALL, LC_CTYPE, and LANG and use the first with a value + to decide the defaults for 8-bit character input and output. Returns + 1 if we set eight-bit mode. */ +int +_rl_init_eightbit () +{ +/* If we have setlocale(3), just check the current LC_CTYPE category + value, and go into eight-bit mode if it's not C or POSIX. */ +#if defined (HAVE_SETLOCALE) + char *lspec, *t; + + t = _rl_init_locale (); /* returns static pointer */ if (t && *t && (t[0] != 'C' || t[1]) && (STREQ (t, "POSIX") == 0)) { @@ -174,9 +186,11 @@ _rl_init_eightbit () _rl_output_meta_chars = 1; break; } + + _rl_utf8locale = *t ? STREQ (t, "utf8") : 0; + xfree (t); return (legal_lang_values[i] ? 1 : 0); - #endif /* !HAVE_SETLOCALE */ } diff --git a/lib/readline/readline.c b/lib/readline/readline.c index 4e88dc76..c022fb1d 100644 --- a/lib/readline/readline.c +++ b/lib/readline/readline.c @@ -1104,7 +1104,7 @@ rl_initialize () { /* If we have never been called before, initialize the terminal and data structures. */ - if (!rl_initialized) + if (rl_initialized == 0) { RL_SETSTATE(RL_STATE_INITIALIZING); readline_initialize_everything (); @@ -1112,6 +1112,8 @@ rl_initialize () rl_initialized++; RL_SETSTATE(RL_STATE_INITIALIZED); } + else + (void)_rl_init_locale (); /* check current locale */ /* Initialize the current line information. */ _rl_init_line_state (); diff --git a/lib/readline/rlmbutil.h b/lib/readline/rlmbutil.h index 4748a610..2d083b9f 100644 --- a/lib/readline/rlmbutil.h +++ b/lib/readline/rlmbutil.h @@ -172,6 +172,8 @@ _rl_wcwidth (wc) # define IS_COMBINING_CHAR(x) (WCWIDTH(x) == 0) #endif +#define UTF8_SINGLEBYTE(c) (((c) & 0x80) == 0) + #else /* !HANDLE_MULTIBYTE */ #undef MB_LEN_MAX @@ -196,6 +198,8 @@ _rl_wcwidth (wc) #define MB_INVALIDCH(x) (0) #define MB_NULLWCH(x) (0) +#define UTF8_SINGLEBYTE(c) (1) + #endif /* !HANDLE_MULTIBYTE */ extern int rl_byte_oriented; diff --git a/lib/readline/rlprivate.h b/lib/readline/rlprivate.h index f2d66163..1269e5c4 100644 --- a/lib/readline/rlprivate.h +++ b/lib/readline/rlprivate.h @@ -331,6 +331,7 @@ extern void _rl_set_insert_mode PARAMS((int, int)); extern void _rl_revert_all_lines PARAMS((void)); /* nls.c */ +extern char *_rl_init_locale PARAMS((void)); extern int _rl_init_eightbit PARAMS((void)); /* parens.c */ diff --git a/lib/readline/text.c b/lib/readline/text.c index c353252b..e57344ad 100644 --- a/lib/readline/text.c +++ b/lib/readline/text.c @@ -695,6 +695,12 @@ _rl_insert_char (count, c) incoming[1] = '\0'; incoming_length = 1; } + else if (_rl_utf8locale && (c & 0x80) == 0) + { + incoming[0] = c; + incoming[1] = '\0'; + incoming_length = 1; + } else { wchar_t wc; @@ -739,6 +745,12 @@ _rl_insert_char (count, c) effect of mbstate is undefined. */ memset (&ps, 0, sizeof (mbstate_t)); } + else if (ret == 1) + { + incoming[0] = pending_bytes[0]; + incoming[incoming_length = 1] = '\0'; + pending_bytes_length = 0; + } else { /* We successfully read a single multibyte character. */ @@ -761,8 +773,13 @@ _rl_insert_char (count, c) i = 0; while (i < string_size) { - strncpy (string + i, incoming, incoming_length); - i += incoming_length; + if (incoming_length == 1) + string[i++] = *incoming; + else + { + strncpy (string + i, incoming, incoming_length); + i += incoming_length; + } } incoming_length = 0; stored_count = 0; @@ -790,8 +807,13 @@ _rl_insert_char (count, c) i = 0; while (i < string_size) { - strncpy (string + i, incoming, incoming_length); - i += incoming_length; + if (incoming_length == 1) + string[i++] = *incoming; + else + { + strncpy (string + i, incoming, incoming_length); + i += incoming_length; + } } while (count) diff --git a/locale.c b/locale.c index 7aa150aa..c42046bb 100644 --- a/locale.c +++ b/locale.c @@ -45,6 +45,7 @@ extern int errno; int locale_utf8locale; /* set but unused for now */ int locale_mb_cur_max; /* value of MB_CUR_MAX for current locale (LC_CTYPE) */ +int locale_shiftstates; extern int dump_translatable_strings, dump_po_strings; @@ -85,6 +86,7 @@ set_default_locale () locale_mb_cur_max = MB_CUR_MAX; locale_utf8locale = locale_isutf8 (default_locale); + locale_shiftstates = mblen ((char *)NULL, 0); } /* Set default values for LC_CTYPE, LC_COLLATE, LC_MESSAGES, LC_NUMERIC and @@ -105,6 +107,7 @@ set_default_locale_vars () locale_setblanks (); locale_mb_cur_max = MB_CUR_MAX; locale_utf8locale = locale_isutf8 (lc_all); + locale_shiftstates = mblen ((char *)NULL, 0); u32reset (); } # endif @@ -208,6 +211,7 @@ set_locale_var (var, value) /* if LC_ALL == "", reset_locale_vars has already called this */ if (*lc_all && x) locale_utf8locale = locale_isutf8 (lc_all); + locale_shiftstates = mblen ((char *)NULL, 0); u32reset (); return r; #else @@ -227,6 +231,7 @@ set_locale_var (var, value) /* if setlocale() returns NULL, the locale is not changed */ if (x) locale_utf8locale = locale_isutf8 (x); + locale_shiftstates = mblen ((char *)NULL, 0); u32reset (); } # endif @@ -363,6 +368,7 @@ reset_locale_vars () locale_mb_cur_max = MB_CUR_MAX; if (x) locale_utf8locale = locale_isutf8 (x); + locale_shiftstates = mblen ((char *)NULL, 0); u32reset (); #endif return 1; diff --git a/patchlevel.h b/patchlevel.h index 1cd7c96c..1bc098b8 100644 --- a/patchlevel.h +++ b/patchlevel.h @@ -25,6 +25,6 @@ regexp `^#define[ ]*PATCHLEVEL', since that's what support/mkversion.sh looks for to find the patch level (for the sccs version string). */ -#define PATCHLEVEL 0 +#define PATCHLEVEL 5 #endif /* _PATCHLEVEL_H_ */ diff --git a/shell.h b/shell.h index 6a6bdcdb..a8ca731f 100644 --- a/shell.h +++ b/shell.h @@ -99,6 +99,7 @@ extern int subshell_environment; extern int shell_compatibility_level; extern int locale_mb_cur_max; +extern int locale_utf8locale; /* Structure to pass around that holds a bitmap of file descriptors to close, and the size of that structure. Used in execute_cmd.c. */ diff --git a/tests/RUN-ONE-TEST b/tests/RUN-ONE-TEST index d29259a9..a5ab273d 100755 --- a/tests/RUN-ONE-TEST +++ b/tests/RUN-ONE-TEST @@ -1,4 +1,4 @@ -BUILD_DIR=/usr/local/build/chet/bash/bash-current +BUILD_DIR=/usr/local/build/bash/bash-current THIS_SH=$BUILD_DIR/bash PATH=$PATH:$BUILD_DIR diff --git a/tests/glob.right b/tests/glob.right index 68c0666f..330f6560 100644 --- a/tests/glob.right +++ b/tests/glob.right @@ -9,7 +9,7 @@ argv[1] = argv[1] = argv[1] = -0000000 141 243 134 142 +0000000 141 243 134 142 0000004 ok 6 ok 7 diff --git a/tests/glob2.sub b/tests/glob2.sub index 2a743409..d33013b6 100644 --- a/tests/glob2.sub +++ b/tests/glob2.sub @@ -1,3 +1,5 @@ +. ./test-glue-functions + var='ab\' case $var in @@ -34,7 +36,7 @@ recho "$b" unset a b v recho "a${alpha}b" -printf "%s" "a${alpha}b" | LC_ALL=C od -b +printf "%s" "a${alpha}b" | LC_ALL=C od -b | _intl_normalize_spaces a=$'\u3b1' [[ $a = $a ]] && echo ok 6