Subject: [PATCH] grep: fix the different behaviour for a invalid sequence between KWset and DFA
From: Norihiro Tanaka <[email-masked]>
To: [email-masked]
Date: Thu, 01 May 2014 00:02:19 +0900
List-Id: <debbugs-submit.debbugs.gnu.org>

Thare is different behaviour for a invalid sequence between KWset and DFA.

  encode() { echo "$1" | tr ABC '\357\274\241'; }
  encode ABC | env LC_ALL=en_US.utf8 src/grep "$(encode A)\|q"
  encode ABC | env LC_ALL=en_US.utf8 src/grep -F "$(encode A)"
  encode sABC | env LC_ALL=en_US.utf8 src/grep "a$(encode A)\|q"
  encode sABC | env LC_ALL=en_US.utf8 src/grep -F "a$(encode A)"

We expect that all of them are same results, but only 4th returns 1 row.
This patch fixes it, changes all into 1 row returned.

Norihiro
From 0b5084286c23e75139cc09e02c1ad8495059eb38 Mon Sep 17 00:00:00 2001
From: Norihiro Tanaka <[email-masked]>
Date: Wed, 30 Apr 2014 11:22:27 +0900
Subject: [PATCH] grep: fix the different behaviour for a invalid sequence
 between KWset and DFA

* src/dfa.c (ctok): Define new global variable.
(dfambcache): Don't cache invalid sequences, because it can't be
expressed with any wide character.
(mbs_to_wchar): Return WEOF for invalid sequences.
(parse_bracket_exp): Fix it.
(lex): Set `ctok'.
(atom, match_anychar, match_mb_charset): Fix it.
* src/searchutils.c (is_mb_middle): Fix it.
* tests/prefix-of-multibyte: Fix it.
---
 src/dfa.c                 | 101 ++++++++++++++++++++++++++++------------------
 src/kwsearch.c            |   7 +---
 src/searchutils.c         |   2 +-
 tests/prefix-of-multibyte |  12 ++++--
 4 files changed, 72 insertions(+), 50 deletions(-)

diff --git a/src/dfa.c b/src/dfa.c
index 362de2c..c83a940 100644
--- a/src/dfa.c
+++ b/src/dfa.c
@@ -456,9 +456,13 @@ dfambcache (struct dfa *d)
       wint_t wi;
       switch (mbrtowc (&wc, &c, 1, &s))
         {
-        default: wi = wc; break;
-        case (size_t) -2: wi = WEOF; break;
-        case (size_t) -1: wi = uc; break;
+        default:
+          wi = wc;
+          break;
+        case (size_t) -1:
+        case (size_t) -2:
+          wi = WEOF;
+          break;
         }
       d->mbrtowc_cache[uc] = wi;
     }
@@ -492,7 +496,6 @@ mbs_to_wchar (wchar_t *pwc, char const *s, size_t n, struct dfa *d)
       if (0 < nbytes && nbytes < (size_t) -2)
         return nbytes;
       memset (&d->mbs, 0, sizeof d->mbs);
-      wc = uc;
     }
 
   *pwc = wc;
@@ -847,6 +850,8 @@ static int cur_mb_len = 1;      /* Length of the multibyte representation of
 /* These variables are used only if (MB_CUR_MAX > 1).  */
 static wchar_t wctok;           /* Wide character representation of the current
                                    multibyte character.  */
+static unsigned int ctok;       /* Single character representation of the current
+                                   multibyte character.  */

 /* Note that characters become unsigned here.  */
@@ -1128,19 +1133,22 @@ parse_bracket_exp (void)
                      to the pair of ranges, [m-z] [M-Z].  Although this code
                      is wrong in multiple ways, it's never used in practice.
                      FIXME: Remove this (and related) unused code.  */
-                  work_mbc->ranges
-                    = maybe_realloc (work_mbc->ranges, work_mbc->nranges + 2,
-                                     &ranges_al, sizeof *work_mbc->ranges);
-                  work_mbc->ranges[work_mbc->nranges].beg
-                    = case_fold ? towlower (wc) : wc;
-                  work_mbc->ranges[work_mbc->nranges++].end
-                    = case_fold ? towlower (wc2) : wc2;
-
-                  if (case_fold && (iswalpha (wc) || iswalpha (wc2)))
+                  if (wc != WEOF && wc2 != WEOF)
                     {
-                      work_mbc->ranges[work_mbc->nranges].beg = towupper (wc);
+                      work_mbc->ranges
+                        = maybe_realloc (work_mbc->ranges, work_mbc->nranges + 2,
+                                         &ranges_al, sizeof *work_mbc->ranges);
+                      work_mbc->ranges[work_mbc->nranges].beg
+                        = case_fold ? towlower (wc) : wc;
                       work_mbc->ranges[work_mbc->nranges++].end
-                        = towupper (wc2);
+                        = case_fold ? towlower (wc2) : wc2;
+
+                      if (case_fold && (iswalpha (wc) || iswalpha (wc2)))
+                        {
+                          work_mbc->ranges[work_mbc->nranges].beg = towupper (wc);
+                          work_mbc->ranges[work_mbc->nranges++].end
+                            = towupper (wc2);
+                        }
                     }
                 }
               else if (using_simple_locale ())
@@ -1184,23 +1192,28 @@ parse_bracket_exp (void)
           continue;
         }
 
-      if (case_fold)
-        {
-          wchar_t folded[CASE_FOLDED_BUFSIZE];
-          int i, n = case_folded_counterparts (wc, folded);
-          work_mbc->chars = maybe_realloc (work_mbc->chars,
-                                           work_mbc->nchars + n, &chars_al,
-                                           sizeof *work_mbc->chars);
-          for (i = 0; i < n; i++)
-            if (!setbit_wc (folded[i], ccl))
-              work_mbc->chars[work_mbc->nchars++] = folded[i];
-        }
-      if (!setbit_wc (wc, ccl))
+      if (wc != WEOF)
         {
-          work_mbc->chars = maybe_realloc (work_mbc->chars, work_mbc->nchars,
-                                           &chars_al, sizeof *work_mbc->chars);
-          work_mbc->chars[work_mbc->nchars++] = wc;
+          if (case_fold)
+            {
+              wchar_t folded[CASE_FOLDED_BUFSIZE];
+              int i, n = case_folded_counterparts (wc, folded);
+              work_mbc->chars = maybe_realloc (work_mbc->chars,
+                                               work_mbc->nchars + n, &chars_al,
+                                               sizeof *work_mbc->chars);
+              for (i = 0; i < n; i++)
+                if (!setbit_wc (folded[i], ccl))
+                  work_mbc->chars[work_mbc->nchars++] = folded[i];
+            }
+          else if (!setbit_wc (wc, ccl))
+            {
+              work_mbc->chars = maybe_realloc (work_mbc->chars, work_mbc->nchars,
+                                               &chars_al, sizeof *work_mbc->chars);
+              work_mbc->chars[work_mbc->nchars++] = wc;
+            }
         }
+      else
+        setbit (c, ccl);
     }
   while ((wc = wc1, (c = c1) != ']'));
 
@@ -1245,7 +1258,8 @@ lex (void)
      "if (backslash) ...".  */
   for (i = 0; i < 2; ++i)
     {
-      FETCH_WC (c, wctok, NULL);
+      FETCH_WC (ctok, wctok, NULL);
+      c = ctok;
       if (c == (unsigned int) EOF)
         goto normal_char;
 
@@ -1776,18 +1790,23 @@ atom (void)
 {
   if (tok == WCHAR)
     {
-      addtok_wc (wctok);
-
-      if (case_fold)
+      if (wctok != WEOF)
         {
-          wchar_t folded[CASE_FOLDED_BUFSIZE];
-          int i, n = case_folded_counterparts (wctok, folded);
-          for (i = 0; i < n; i++)
+          addtok_wc (wctok);
+
+          if (case_fold)
             {
-              addtok_wc (folded[i]);
-              addtok (OR);
+              wchar_t folded[CASE_FOLDED_BUFSIZE];
+              int i, n = case_folded_counterparts (wctok, folded);
+              for (i = 0; i < n; i++)
+                {
+                  addtok_wc (folded[i]);
+                  addtok (OR);
+                }
             }
         }
+      else
+        addtok_mb (ctok, 3);
 
       tok = lex ();
     }
@@ -2949,6 +2968,8 @@ match_anychar (struct dfa *d, state_num s, position pos,
       if (syntax_bits & RE_DOT_NOT_NULL)
         return 0;
     }
+  else if (wc == WEOF)
+    return 0;
 
   context = wchar_context (wc);
   if (!SUCCEEDS_IN_CONTEXT (pos.constraint, d->states[s].context, context))
@@ -2985,6 +3006,8 @@ match_mb_charset (struct dfa *d, state_num s, position pos,
       if (syntax_bits & RE_DOT_NOT_NULL)
         return 0;
     }
+  else if (wc == WEOF)
+    return 0;
 
   context = wchar_context (wc);
   if (!SUCCEEDS_IN_CONTEXT (pos.constraint, d->states[s].context, context))
diff --git a/src/kwsearch.c b/src/kwsearch.c
index 7c64c86..46569e9 100644
--- a/src/kwsearch.c
+++ b/src/kwsearch.c
@@ -131,12 +131,7 @@ Fexecute (char const *buf, size_t size, size_t *match_size,
         {
           /* The match was a part of multibyte character, advance at least
              one byte to ensure no infinite loop happens.  */
-          mbstate_t s;
-          memset (&s, 0, sizeof s);
-          size_t mb_len = mbrlen (mb_start, (buf + size) - (beg + offset), &s);
-          if (mb_len == (size_t) -2 || mb_len == (size_t) -1)
-            goto failure;
-          beg = mb_start + mb_len - 1;
+          beg = mb_start;
           continue;
         }
       beg += offset;
diff --git a/src/searchutils.c b/src/searchutils.c
index 6440f07..ea26a70 100644
--- a/src/searchutils.c
+++ b/src/searchutils.c
@@ -270,5 +270,5 @@ is_mb_middle (const char **good, const char *buf, const char *end,
     return true;
 
   /* P == BUF here.  */
-  return 0 < match_len && match_len < mbrlen (p, end - p, &cur_state);
+  return false;
 }
diff --git a/tests/prefix-of-multibyte b/tests/prefix-of-multibyte
index b15fa9b..70a924e 100755
--- a/tests/prefix-of-multibyte
+++ b/tests/prefix-of-multibyte
@@ -1,5 +1,5 @@
 #!/bin/sh
-# This would mistakenly print a line prior to grep-2.6.2.
+# This would mistakenly print a line prior to grep-2.18.
 . "${srcdir=.}/init.sh"; path_prepend_ ../src
 
 require_en_utf8_locale_
@@ -7,14 +7,18 @@ require_compiled_in_MB_support
 
 encode() { echo "$1" | tr ABC '\357\274\241'; }
 
+encode ABC >exp1
+encode aABC >exp2
+
 fail=0
 
 for LOC in en_US.UTF-8 $LOCALE_FR_UTF8; do
   for opt in '' '-F'; do
     out=out-$opt-$LOC
-    encode ABC | LC_ALL=$LOC grep $opt "$(encode A)" > $out 2>&1
-    test $? = 1 || fail=1
-    compare /dev/null $out || fail=1
+    LC_ALL=$LOC grep $opt "$(encode A)" exp1 >$out || fail=1
+    compare exp1 $out || fail=1
+    LC_ALL=$LOC grep $opt "$(encode aA)" exp2 >$out || fail=1
+    compare exp2 $out || fail=1
   done
 done
 
-- 
1.9.2