Subject: Re: bug#19833: grep: internal PCRE error: -27 since updating from 2.16 to 2.21
From: Paul Eggert <[email-masked]>
To: Oliver Freyermuth <[email-masked]>, [email-masked]
Date: Tue, 10 Feb 2015 23:49:17 -0800
List-Id: <debbugs-submit.debbugs.gnu.org>

Thanks for the bug report and test case.  I installed the attached patch, which 
fixed the bug for me, and which adds a similar test case to the test database.
From 675dccfdaa14ff5513b5b652da8d9286e65b69e2 Mon Sep 17 00:00:00 2001
From: Paul Eggert <[email-masked]>
Date: Tue, 10 Feb 2015 23:44:36 -0800
Subject: [PATCH] Grow the JIT stack if it becomes exhausted

Problem reported by Oliver Freyermuth in: http://bugs.gnu.org/19833
* NEWS: Document the fix.
* tests/Makefile.am (TESTS): Add pcre-jitstack.
* tests/pcre-jitstack: New file.
* src/pcresearch.c (NSUB): Move decl earlier, since it's needed
earlier now.
(jit_stack_size) [PCRE_STUDY_JIT_COMPILE]: New static var.
(jit_exec): New function.
(Pcompile): Initialize jit_stack_size.
(Pexecute): Use new jit_exec function.  Report a useful diagnostic
if the error is PCRE_ERROR_JIT_STACKLIMIT.
---
 NEWS                |  3 +++
 src/pcresearch.c    | 73 +++++++++++++++++++++++++++++++++++++++--------------
 tests/Makefile.am   |  1 +
 tests/pcre-jitstack | 39 ++++++++++++++++++++++++++++
 4 files changed, 97 insertions(+), 19 deletions(-)
 create mode 100755 tests/pcre-jitstack

diff --git a/NEWS b/NEWS
index da8bc78..071caab 100644
--- a/NEWS
+++ b/NEWS
@@ -7,6 +7,9 @@ GNU grep NEWS                                    -*- outline -*-
   grep no longer reads from uninitialized memory or from beyond the end
   of the heap-allocated input buffer.  This fix addressed CVE-2015-1345.
 
+  When the JIT stack is exhausted, grep -P now grows the stack rather
+  than reporting an internal PCRE error.
+
 
 * Noteworthy changes in release 2.21 (2014-11-23) [stable]
 
diff --git a/src/pcresearch.c b/src/pcresearch.c
index 0bd021a..afe4f2b 100644
--- a/src/pcresearch.c
+++ b/src/pcresearch.c
@@ -27,6 +27,11 @@
 #endif
 
 #if HAVE_LIBPCRE
+
+/* This must be at least 2; everything after that is for performance
+   in pcre_exec.  */
+enum { NSUB = 300 };
+
 /* Compiled internal form of a Perl regular expression.  */
 static pcre *cre;
 
@@ -36,6 +41,45 @@ static pcre_extra *extra;
 # ifndef PCRE_STUDY_JIT_COMPILE
 #  define PCRE_STUDY_JIT_COMPILE 0
 # endif
+
+# if PCRE_STUDY_JIT_COMPILE
+/* Maximum size of the JIT stack.  */
+static int jit_stack_size;
+# endif
+
+/* Match the already-compiled PCRE pattern against the data in P, of
+   size SEARCH_BYTES, with options OPTIONS, and storing resulting
+   matches into SUB.  Return the (nonnegative) match location or a
+   (negative) error number.  */
+static int
+jit_exec (char const *p, int search_bytes, int options, int *sub)
+{
+  while (true)
+    {
+      int e = pcre_exec (cre, extra, p, search_bytes, 0, options, sub, NSUB);
+
+# if PCRE_STUDY_JIT_COMPILE
+      if (e == PCRE_ERROR_JIT_STACKLIMIT
+          && 0 < jit_stack_size && jit_stack_size <= INT_MAX / 2)
+        {
+          int old_size = jit_stack_size;
+          int new_size = jit_stack_size = old_size * 2;
+          static pcre_jit_stack *jit_stack;
+          if (jit_stack)
+            pcre_jit_stack_free (jit_stack);
+          jit_stack = pcre_jit_stack_alloc (old_size, new_size);
+          if (!jit_stack)
+            error (EXIT_TROUBLE, 0,
+                   _("failed to allocate memory for the PCRE JIT stack"));
+          pcre_assign_jit_stack (extra, NULL, jit_stack);
+          continue;
+        }
+# endif
+
+      return e;
+    }
+}
+
 #endif
 
 #if HAVE_LIBPCRE
@@ -44,10 +88,6 @@ static pcre_extra *extra;
 static int empty_match[2];
 #endif
 
-/* This must be at least 2; everything after that is for performance
-   in pcre_exec.  */
-enum { NSUB = 300 };
-
 void
 Pcompile (char const *pattern, size_t size)
 {
@@ -121,19 +161,11 @@ Pcompile (char const *pattern, size_t size)
   if (pcre_fullinfo (cre, extra, PCRE_INFO_JIT, &e))
     error (EXIT_TROUBLE, 0, _("internal error (should never happen)"));
 
+  /* The PCRE documentation says that a 32 KiB stack is the default.  */
   if (e)
-    {
-      /* A 32K stack is allocated for the machine code by default, which
-         can grow to 512K if necessary. Since JIT uses far less memory
-         than the interpreter, this should be enough in practice.  */
-      pcre_jit_stack *jit_stack = pcre_jit_stack_alloc (32 * 1024, 512 * 1024);
-      if (!jit_stack)
-        error (EXIT_TROUBLE, 0,
-               _("failed to allocate memory for the PCRE JIT stack"));
-      pcre_assign_jit_stack (extra, NULL, jit_stack);
-    }
-
+    jit_stack_size = 32 << 10;
 # endif
+
   free (re);
 
   int sub[NSUB];
@@ -214,8 +246,7 @@ Pexecute (char const *buf, size_t size, size_t *match_size,
           if (multiline)
             options |= PCRE_NO_UTF8_CHECK;
 
-          e = pcre_exec (cre, extra, p, search_bytes, 0,
-                         options, sub, NSUB);
+          e = jit_exec (p, search_bytes, options, sub);
           if (e != PCRE_ERROR_BADUTF8)
             {
               if (0 < e && multiline && sub[1] - sub[0] != 0)
@@ -268,9 +299,13 @@ Pexecute (char const *buf, size_t size, size_t *match_size,
         case PCRE_ERROR_NOMEMORY:
           error (EXIT_TROUBLE, 0, _("memory exhausted"));
 
+# if PCRE_STUDY_JIT_COMPILE
+        case PCRE_ERROR_JIT_STACKLIMIT:
+          error (EXIT_TROUBLE, 0, _("PCRE JIT stack exhausted"));
+# endif
+
         case PCRE_ERROR_MATCHLIMIT:
-          error (EXIT_TROUBLE, 0,
-                 _("exceeded PCRE's backtracking limit"));
+          error (EXIT_TROUBLE, 0, _("exceeded PCRE's backtracking limit"));
 
         default:
           /* For now, we lump all remaining PCRE failures into this basket.
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 0508cd2..8fcf8f6 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -95,6 +95,7 @@ TESTS =						\
   pcre-abort					\
   pcre-infloop					\
   pcre-invalid-utf8-input			\
+  pcre-jitstack					\
   pcre-o					\
   pcre-utf8					\
   pcre-w					\
diff --git a/tests/pcre-jitstack b/tests/pcre-jitstack
new file mode 100755
index 0000000..03587f1
--- /dev/null
+++ b/tests/pcre-jitstack
@@ -0,0 +1,39 @@
+#! /bin/sh
+# Grep 2.21 would report "grep: internal PCRE error: -27"
+#
+# Copyright 2015 Free Software Foundation, Inc.
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+require_pcre_
+
+nl_base64=$(echo | (base64) 2>/dev/null) && test "X$nl_base64" = XCg== ||
+  skip_ "your system lacks the base64 program"
+foo=$( (echo foo | gzip | gzip -d) 2>/dev/null) && test "X$foo" = Xfoo ||
+  skip_ "your system lacks the gzip program"
+
+fail=0
+
+base64 -d >pcrejit.txt.gz <<'EOF'
+EOF
+test $? -eq 0 || framework_failure_
+
+gzip -d pcrejit.txt || framework_failure_
+
+LC_ALL=C grep -P -n '^([/](?!/)|[^/])*~/.*' pcrejit.txt
+test $? -eq 1 || fail=1
+
+Exit $fail
-- 
2.1.0