Subject: Re: bug#17072: dfa change apparently needed on Irix

From: Paul Eggert <[email-masked]>
Date: Sun, 06 Apr 2014 11:00:58 -0700

Thanks to Nelson H.F. Beebe's machines I've reproduced Bug#17072 on IRIX 
and have verified that the attached gawk patch fixes it.  This patch is 
almost identical to the gawk patch I submitted in 
<http://bugs.gnu.org/17157#44>.  It improves on the earlier patch only 
by changing gawk's dfa.c to look more like grep's dfa.c in one more way: 
include "dfa.h" before all other include files (except config.h).

I'll CC: this to Bug#17157 so that the patch is visible there too.  No 
further change should be needed to grep for this.
From 4622d51104feb47f1379e8d8fd909b1dd42598a3 Mon Sep 17 00:00:00 2001
From: Paul Eggert <[email-masked]>
Date: Thu, 3 Apr 2014 18:04:52 -0700
Subject: [PATCH] awk: simplify dfa.c by having it not include mbsupport.h
 directly

This syncs dfa.c better with 'grep'.
* awk.h, regex_internal.h, dfa.c: Don't include mbsupport.h.
* custom.h: Include mbsupport.h here instead.
(_GL_ATTRIBUTE_PURE): Move here from dfa.c, to lessen the
number of differences between grep's dfa.c and ours.
* dfa.c: Include dfa.h first; that works now.
Include wchar.h and wctype.h unconditionally, as
this simplifies the use of dfa.c in grep, and it does no harm
in gawk.
(gawk_mb_cur_max, MB_CUR_MAX, mbrtowc) [LIBC_IS_BORKED]:
Move to mbsupport.h (needed for consistency in all uses),
and fix mbrtowc to return size_t.
(setlocale, static_assert): Likewise.
(struct dfa, dfambcache, mbs_to_wchar)
(is_valid_unibyte_character, setbit_wc, using_utf8, FETCH_WC)
(addtok_wc, add_utf8_anychar, atom, state_index, epsclosure)
(dfaanalyze, dfastate, prepare_wc_buf, dfaoptimize, dfafree, dfamust):
* dfasearch.c (EGexecute):
* grep.c (main):
* searchutils.c (mbtoupper):
Assume MBS_SUPPORT.
* mbsupport.h [!MBS_SUPPORT]: Include wchar.h, wctype.h
before overriding their definitions.
(WEOF, towupper, towlower, btowc, iswalnum, iswalpha, iswupper)
(iswlower, mbrtowc, wcrtomb, wctype, iswctype, wcscoll):
(btowc): Parenthesize properly.
(mbrtowc, wcrtomb): New macros.
(wctype, iswctype, wcscoll): Define to gawk_wctype etc. to avoid
collisions with standard library.
* missing_d/wcmisc.c: Remove now-unnecessary ifdefs.
---
 ChangeLog           |  32 +++++++++++++++
 awk.h               |   2 -
 custom.h            |   9 ++++
 dfa.c               | 115 ++++++++--------------------------------------------
 mbsupport.h         |  54 ++++++++++++++++++++++--
 missing_d/ChangeLog |   4 ++
 missing_d/wcmisc.c  |  10 -----
 regex_internal.h    |   2 -
 8 files changed, 112 insertions(+), 116 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index a0efd89..36fb0f4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,35 @@
+2014-04-03  Paul Eggert  <[email-masked]>
+
+	awk: simplify dfa.c by having it not include mbsupport.h directly
+	This syncs dfa.c better with 'grep'.
+	* awk.h, regex_internal.h, dfa.c: Don't include mbsupport.h.
+	* custom.h: Include mbsupport.h here instead.
+	(_GL_ATTRIBUTE_PURE): Move here from dfa.c, to lessen the
+	number of differences between grep's dfa.c and ours.
+	* dfa.c: Include wchar.h and wctype.h unconditionally, as
+	this simplifies the use of dfa.c in grep, and it does no harm
+	in gawk.
+	(gawk_mb_cur_max, MB_CUR_MAX, mbrtowc) [LIBC_IS_BORKED]:
+	Move to mbsupport.h (needed for consistency in all uses),
+	and fix mbrtowc to return size_t.
+	(struct dfa, dfambcache, mbs_to_wchar)
+	(is_valid_unibyte_character, setbit_wc, using_utf8, FETCH_WC)
+	(addtok_wc, add_utf8_anychar, atom, state_index, epsclosure)
+	(dfaanalyze, dfastate, prepare_wc_buf, dfaoptimize, dfafree, dfamust):
+	* dfasearch.c (EGexecute):
+	* grep.c (main):
+	* searchutils.c (mbtoupper):
+	Assume MBS_SUPPORT.
+	* mbsupport.h [!MBS_SUPPORT]: Include wchar.h, wctype.h
+	before overriding their definitions.
+	(WEOF, towupper, towlower, btowc, iswalnum, iswalpha, iswupper)
+	(iswlower, mbrtowc, wcrtomb, wctype, iswctype, wcscoll):
+	#undef before #defining.
+	(btowc): Parenthesize properly.
+	(mbrtowc, wcrtomb): New macros.
+	(wctype, iswctype, wcscoll): Define to gawk_wctype etc. to avoid
+	collisions with standard library.
+
 2014-04-03         Arnold D. Robbins     <[email-masked]>
 
 	* regcomp.c (parse_bracket_exp): Move a call to `re_free' inside
diff --git a/awk.h b/awk.h
index aefdd07..cdba7a8 100644
--- a/awk.h
+++ b/awk.h
@@ -95,8 +95,6 @@ extern int errno;
 #include "missing_d/gawkbool.h"
 #endif
 
-#include "mbsupport.h" /* defines MBS_SUPPORT */
-
 #if MBS_SUPPORT
 /* We can handle multibyte strings.  */
 #include <wchar.h>
diff --git a/custom.h b/custom.h
index 36b4aa0..bade4cf 100644
--- a/custom.h
+++ b/custom.h
@@ -76,3 +76,12 @@
 extern int setenv(const char *name, const char *value, int rewrite);
 extern int unsetenv(const char *name);
 #endif
+
+/* The __pure__ attribute was added in gcc 2.96.  */
+#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)
+# define _GL_ATTRIBUTE_PURE __attribute__ ((__pure__))
+#else
+# define _GL_ATTRIBUTE_PURE /* empty */
+#endif
+
+#include "mbsupport.h"
diff --git a/dfa.c b/dfa.c
index 378305d..b5f68ba 100644
--- a/dfa.c
+++ b/dfa.c
@@ -22,6 +22,8 @@
 
 #include <config.h>
 
+#include "dfa.h"
+
 #include <assert.h>
 #include <ctype.h>
 #include <stdio.h>
@@ -43,16 +45,6 @@
 #include "missing_d/gawkbool.h"
 #endif /* HAVE_STDBOOL_H */
 
-/* Gawk doesn't use Gnulib, so don't assume that setlocale and
-   static_assert are present.  */
-#ifndef LC_ALL
-# define setlocale(category, locale) NULL
-#endif
-#ifndef static_assert
-# define static_assert(cond, diagnostic) \
-    extern int (*foo (void)) [!!sizeof (struct { int foo: (cond) ? 8 : -1; })]
-#endif
-
 #define STREQ(a, b) (strcmp (a, b) == 0)
 
 /* ISASCIIDIGIT differs from isdigit, as follows:
@@ -69,21 +61,8 @@
 #include "gettext.h"
 #define _(str) gettext (str)
 
-#include "mbsupport.h" /* Define MBS_SUPPORT to 1 or 0, as appropriate.  */
-#if MBS_SUPPORT
-/* We can handle multibyte strings.  */
-# include <wchar.h>
-# include <wctype.h>
-#endif
-
-#ifdef GAWK
-/* The __pure__ attribute was added in gcc 2.96.  */
-#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)
-# define _GL_ATTRIBUTE_PURE __attribute__ ((__pure__))
-#else
-# define _GL_ATTRIBUTE_PURE /* empty */
-#endif
-#endif /* GAWK */
+#include <wchar.h>
+#include <wctype.h>
 
 #if HAVE_LANGINFO_CODESET
 # include <langinfo.h>
@@ -91,8 +70,6 @@
 
 #include "xalloc.h"
 
-#include "dfa.h"
-
 #ifdef GAWK
 static int
 is_blank (int c)
@@ -101,14 +78,6 @@ is_blank (int c)
 }
 #endif /* GAWK */
 
-#ifdef LIBC_IS_BORKED
-extern int gawk_mb_cur_max;
-#undef MB_CUR_MAX
-#define MB_CUR_MAX gawk_mb_cur_max
-#undef mbrtowc
-#define mbrtowc(a, b, c, d) (-1)
-#endif
-
 /* HPUX defines these as macros in sys/param.h.  */
 #ifdef setbit
 # undef setbit
@@ -412,13 +381,11 @@ struct dfa
   size_t nmultibyte_prop;
   int *multibyte_prop;
 
-#if MBS_SUPPORT
   /* A table indexed by byte values that contains the corresponding wide
      character (if any) for that byte.  WEOF means the byte is the
      leading byte of a multibyte character.  Invalid and null bytes are
      mapped to themselves.  */
   wint_t mbrtowc_cache[NOTCHAR];
-#endif
 
   /* Array of the bracket expression in the DFA.  */
   struct mb_char_classes *mbcsets;
@@ -525,7 +492,6 @@ static void regexp (void);
 static void
 dfambcache (struct dfa *d)
 {
-#if MBS_SUPPORT
   int i;
   for (i = CHAR_MIN; i <= CHAR_MAX; ++i)
     {
@@ -542,10 +508,8 @@ dfambcache (struct dfa *d)
         }
       d->mbrtowc_cache[uc] = wi;
     }
-#endif
 }
 
-#if MBS_SUPPORT
 /* Given the dfa D, store into *PWC the result of converting the
    leading bytes of the multibyte buffer S of length N bytes, updating
    the conversion state in *MBS.  On conversion error, convert just a
@@ -579,7 +543,6 @@ mbs_to_wchar (struct dfa *d, wchar_t *pwc, char const *s, size_t n,
   *pwc = wc;
   return 1;
 }
-#endif
 
 #ifdef DEBUG
 
@@ -749,7 +712,7 @@ static charclass newline;
 #ifdef __GLIBC__
 # define is_valid_unibyte_character(c) 1
 #else
-# define is_valid_unibyte_character(c) (! (MBS_SUPPORT && btowc (c) == WEOF))
+# define is_valid_unibyte_character(c) (btowc (c) != WEOF)
 #endif
 
 /* Return non-zero if C is a "word-constituent" byte; zero otherwise.  */
@@ -810,17 +773,12 @@ dfasyntax (reg_syntax_t bits, int fold, unsigned char eol)
 static bool
 setbit_wc (wint_t wc, charclass c)
 {
-#if MBS_SUPPORT
   int b = wctob (wc);
   if (b == EOF)
     return false;
 
   setbit (b, c);
   return true;
-#else
-  abort ();
-   /*NOTREACHED*/ return false;
-#endif
 }
 
 /* Set a bit for B and its case variants in the charclass C.
@@ -845,7 +803,7 @@ using_utf8 (void)
   static int utf8 = -1;
   if (utf8 == -1)
     {
-#if defined HAVE_LANGINFO_CODESET && MBS_SUPPORT
+#if defined HAVE_LANGINFO_CODESET
       utf8 = (STREQ (nl_langinfo (CODESET), "UTF-8"));
 #else
       utf8 = 0;
@@ -938,7 +896,6 @@ static unsigned char const *buf_begin;  /* reference to begin in dfaexec.  */
 static unsigned char const *buf_end;    /* reference to end in dfaexec.  */

-#if MBS_SUPPORT
 /* Note that characters become unsigned here.  */
 # define FETCH_WC(c, wc, eoferr)		\
   do {						\
@@ -961,23 +918,6 @@ static unsigned char const *buf_end;    /* reference to end in dfaexec.  */
       }						\
   } while (0)
 
-#else
-/* Note that characters become unsigned here.  */
-# define FETCH_WC(c, unused, eoferr)  \
-  do {				      \
-    if (! lexleft)		      \
-      {				      \
-        if ((eoferr) != 0)	      \
-          dfaerror (eoferr);	      \
-        else			      \
-          return lasttok = END;	      \
-      }				      \
-    (c) = to_uchar (*lexptr++);       \
-    --lexleft;			      \
-  } while (0)
-
-#endif /* MBS_SUPPORT */
-
 #ifndef MIN
 # define MIN(a,b) ((a) < (b) ? (a) : (b))
 #endif
@@ -1761,7 +1701,6 @@ addtok (token t)
     }
 }
 
-#if MBS_SUPPORT
 /* We treat a multibyte character as a single atom, so that DFA
    can treat a multibyte character as a single expression.
 
@@ -1793,17 +1732,10 @@ addtok_wc (wint_t wc)
       addtok (CAT);
     }
 }
-#else
-static void
-addtok_wc (wint_t wc)
-{
-}
-#endif
 
 static void
 add_utf8_anychar (void)
 {
-#if MBS_SUPPORT
   static const charclass utf8_classes[5] = {
     {0, 0, 0, 0, ~0, ~0, 0, 0},		/* 80-bf: non-leading bytes */
     {~0, ~0, ~0, ~0, 0, 0, 0, 0},       /* 00-7f: 1-byte sequence */
@@ -1848,7 +1780,6 @@ add_utf8_anychar (void)
       addtok (CAT);
       addtok (OR);
     }
-#endif
 }
 
 /* The grammar understood by the parser is as follows.
@@ -1889,7 +1820,7 @@ add_utf8_anychar (void)
 static void
 atom (void)
 {
-  if (MBS_SUPPORT && tok == WCHAR)
+  if (tok == WCHAR)
     {
       addtok_wc (wctok);
 
@@ -1906,7 +1837,7 @@ atom (void)
 
       tok = lex ();
     }
-  else if (MBS_SUPPORT && tok == ANYCHAR && using_utf8 ())
+  else if (tok == ANYCHAR && using_utf8 ())
     {
       /* For UTF-8 expand the period to a series of CSETs that define a valid
          UTF-8 character.  This avoids using the slow multibyte path.  I'm
@@ -1920,9 +1851,7 @@ atom (void)
     }
   else if ((tok >= 0 && tok < NOTCHAR) || tok >= CSET || tok == BACKREF
            || tok == BEGLINE || tok == ENDLINE || tok == BEGWORD
-#if MBS_SUPPORT
            || tok == ANYCHAR || tok == MBCSET
-#endif /* MBS_SUPPORT */
            || tok == ENDWORD || tok == LIMWORD || tok == NOTLIMWORD)
     {
       addtok (tok);
@@ -2205,11 +2134,9 @@ state_index (struct dfa *d, position_set const *s, int context)
   d->states[i].backref = 0;
   d->states[i].constraint = 0;
   d->states[i].first_end = 0;
-  if (MBS_SUPPORT)
-    {
-      d->states[i].mbps.nelem = 0;
-      d->states[i].mbps.elems = NULL;
-    }
+  d->states[i].mbps.nelem = 0;
+  d->states[i].mbps.elems = NULL;
+
   for (j = 0; j < s->nelem; ++j)
     if (d->tokens[s->elems[j].index] < 0)
       {
@@ -2247,10 +2174,8 @@ epsclosure (position_set * s, struct dfa const *d)
   for (i = 0; i < s->nelem; ++i)
     if (d->tokens[s->elems[i].index] >= NOTCHAR
         && d->tokens[s->elems[i].index] != BACKREF
-#if MBS_SUPPORT
         && d->tokens[s->elems[i].index] != ANYCHAR
         && d->tokens[s->elems[i].index] != MBCSET
-#endif
         && d->tokens[s->elems[i].index] < CSET)
       {
         old = s->elems[i];
@@ -2567,9 +2492,7 @@ dfaanalyze (struct dfa *d, int searchflag)
      it with its epsilon closure.  */
   for (i = 0; i < d->tindex; ++i)
     if (d->tokens[i] < NOTCHAR || d->tokens[i] == BACKREF
-#if MBS_SUPPORT
         || d->tokens[i] == ANYCHAR || d->tokens[i] == MBCSET
-#endif
         || d->tokens[i] >= CSET)
       {
 #ifdef DEBUG
@@ -2679,9 +2602,8 @@ dfastate (state_num s, struct dfa *d, state_num trans[])
         setbit (d->tokens[pos.index], matches);
       else if (d->tokens[pos.index] >= CSET)
         copyset (d->charclasses[d->tokens[pos.index] - CSET], matches);
-      else if (MBS_SUPPORT
-               && (d->tokens[pos.index] == ANYCHAR
-                   || d->tokens[pos.index] == MBCSET))
+      else if (d->tokens[pos.index] == ANYCHAR
+               || d->tokens[pos.index] == MBCSET)
         /* MB_CUR_MAX > 1  */
         {
           /* ANYCHAR and MBCSET must match with a single character, so we
@@ -2855,7 +2777,7 @@ dfastate (state_num s, struct dfa *d, state_num trans[])
       /* If we are building a searching matcher, throw in the positions
          of state 0 as well.  */
       if (d->searchflag
-          && (!MBS_SUPPORT || (d->mb_cur_max == 1 || !next_isnt_1st_byte)))
+          && (d->mb_cur_max == 1 || !next_isnt_1st_byte))
         for (j = 0; j < d->states[0].elems.nelem; ++j)
           insert (d->states[0].elems.elems[j], &follows);
 
@@ -3407,7 +3329,6 @@ transit_state (struct dfa *d, state_num s, unsigned char const **pp)
 static void
 prepare_wc_buf (struct dfa *d, const char *begin, const char *end)
 {
-#if MBS_SUPPORT
   unsigned char eol = eolbyte;
   size_t i;
   size_t ilim = end - begin + 1;
@@ -3431,7 +3352,6 @@ prepare_wc_buf (struct dfa *d, const char *begin, const char *end)
   buf_end = (unsigned char *) (begin + i);
   mblen_buf[i] = 0;
   inputwcs[i] = 0;              /* sentinel */
-#endif /* MBS_SUPPORT */
 }
 
 /* Search through a buffer looking for a match to the given struct dfa.
@@ -3653,7 +3573,7 @@ dfaoptimize (struct dfa *d)
 {
   size_t i;
 
-  if (!MBS_SUPPORT || !using_utf8 ())
+  if (!using_utf8 ())
     return;
 
   for (i = 0; i < d->tindex; ++i)
@@ -3703,8 +3623,7 @@ dfafree (struct dfa *d)
   for (i = 0; i < d->sindex; ++i)
     {
       free (d->states[i].elems.elems);
-      if (MBS_SUPPORT)
-        free (d->states[i].mbps.elems);
+      free (d->states[i].mbps.elems);
     }
   free (d->states);
   for (i = 0; i < d->tindex; ++i)
@@ -4179,7 +4098,7 @@ dfamust (struct dfa *d)
               /* not on *my* shift */
               goto done;
             }
-          else if (t >= CSET || !MBS_SUPPORT || t == ANYCHAR || t == MBCSET)
+          else if (t >= CSET || t == ANYCHAR || t == MBCSET)
             {
               /* easy enough */
               resetmust (mp);
diff --git a/mbsupport.h b/mbsupport.h
index 9a62486..ab33e91 100644
--- a/mbsupport.h
+++ b/mbsupport.h
@@ -66,6 +66,15 @@
 #endif
 
 #if ! MBS_SUPPORT
+
+/* Include wchar.h and wctype.h so their definitions can be overridden.  */
+
+# include <wchar.h>
+# include <wctype.h>
+
+/* Override the definitions of wchar.h and wctype.h to provide a
+   unibyte substitute that is good enough for Gawk.  */
+
 # undef MB_CUR_MAX
 # define MB_CUR_MAX 1
 
@@ -78,15 +87,24 @@
 #define wctype_t	int
 #define wint_t		int
 #define mbstate_t	int
+#undef WEOF
 #define WEOF		EOF
+#undef towupper
 #define towupper	toupper
+#undef towlower
 #define towlower	tolower
 #ifndef __DJGPP__
-#define btowc(x)	((int)x)
+#undef btowc
+#define btowc(x)	((int) (x))
 #endif
+#undef iswalnum
 #define iswalnum	isalnum
+#undef iswalpha
 #define iswalpha	isalpha
+#undef iswupper
 #define iswupper	isupper
+#undef iswlower
+#define iswlower	islower
 #if defined(ZOS_USS)
 #undef towupper
 #undef towlower
@@ -94,12 +112,40 @@
 #undef iswalnum
 #undef iswalpha
 #undef iswupper
-#undef wctype
-#undef iswctype
-#undef wcscoll
 #endif
 
+#undef mbrtowc
+#define mbrtowc(pwc, s, n, ps) ((size_t) -1)
+#undef wcrtomb
+#define wcrtomb(s, wc, ps) ((size_t) -1)
+
+#undef wctype
+#define wctype gawk_wctype
 extern wctype_t wctype(const char *name);
+#undef iswctype
+#define iswctype gawk_iswctype
 extern int iswctype(wint_t wc, wctype_t desc);
+#undef wcscoll
+#define wcscoll gawk_wcscoll
 extern int wcscoll(const wchar_t *ws1, const wchar_t *ws2);
 #endif
+
+#ifdef LIBC_IS_BORKED
+# include <wchar.h>
+extern int gawk_mb_cur_max;
+# undef MB_CUR_MAX
+# undef mbrtowc
+# define MB_CUR_MAX gawk_mb_cur_max
+# define mbrtowc(a, b, c, d) ((size_t) -1)
+#endif
+
+#include <locale.h>
+#ifndef LC_ALL
+# define setlocale(category, locale) NULL
+#endif
+
+#include <assert.h>
+#ifndef static_assert
+# define static_assert(cond, diagnostic) \
+    extern int (*foo (void)) [!!sizeof (struct { int foo: (cond) ? 8 : -1; })]
+#endif
diff --git a/missing_d/ChangeLog b/missing_d/ChangeLog
index f94c070..7fa6541 100644
--- a/missing_d/ChangeLog
+++ b/missing_d/ChangeLog
@@ -1,3 +1,7 @@
+2014-04-03  Paul Eggert  <[email-masked]>
+
+	* wcmisc.c: Remove now-unnecessary ifdefs.
+
 2013-05-09         Arnold D. Robbins     <[email-masked]>
 
 	* 4.1.0: Release tar ball made.
diff --git a/missing_d/wcmisc.c b/missing_d/wcmisc.c
index d2b7aa0..89e24c9 100644
--- a/missing_d/wcmisc.c
+++ b/missing_d/wcmisc.c
@@ -16,7 +16,6 @@
    Foundation, Inc.,
    51 Franklin Street - Fifth Floor, Boston, MA  02110-1301, USA */
 
-#if !defined(HAVE_WCTYPE) || !defined(HAVE_ISWCTYPE)
 static const char *classes[] = {
 	"<dummy>",
 	"alnum",
@@ -33,16 +32,12 @@ static const char *classes[] = {
 	"xdigit",
 	NULL
 };
-#endif
 
-#ifndef HAVE_ISWCTYPE
 static int is_blank (int c)
 {
    return (c == ' ' || c == '\t');
 }
-#endif
 
-#ifndef HAVE_WCTYPE
 wctype_t wctype(const char *name)
 {
 	int i;
@@ -53,9 +48,7 @@ wctype_t wctype(const char *name)
 
 	return 0;
 }
-#endif
 
-#ifndef HAVE_ISWCTYPE
 int iswctype(wint_t wc, wctype_t desc)
 {
 	int j = sizeof(classes) / sizeof(classes[0]);
@@ -79,9 +72,7 @@ int iswctype(wint_t wc, wctype_t desc)
 	default:	return 0;
 	}
 }
-#endif
 
-#ifndef HAVE_WCSCOLL
 int wcscoll(const wchar_t *ws1, const wchar_t *ws2)
 {
 	size_t i;
@@ -95,6 +86,5 @@ int wcscoll(const wchar_t *ws1, const wchar_t *ws2)
 
 	return (ws1[i] - ws2[i]);
 }
-#endif
 
 /*wcmisc.c*/
diff --git a/regex_internal.h b/regex_internal.h
index c8981a0..758cf47 100644
--- a/regex_internal.h
+++ b/regex_internal.h
@@ -26,8 +26,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "mbsupport.h" /* gawk */
-
 #if defined HAVE_LANGINFO_H || defined HAVE_LANGINFO_CODESET || defined _LIBC
 # include <langinfo.h>
 #endif
-- 
1.9.0