Subject: bug#6286: General delimited literals in ruby-mode patch

From: Dmitry Gutov <[email-masked]>
Date: Tue, 14 Aug 2012 07:56:42 +0400

Here are the 3 previously submitted patches as 2 files, slightly 
reworded, and with proper ChangeLog entries.

I think the first one is quite important, and I've had a user emailing 
me about it.
The second one deals with a fairly narrow edge case, so it's kinda optional.

--Dmitry
diff --git a/lisp/ChangeLog b/lisp/ChangeLog
index cedf854..4f52796 100644
--- a/lisp/ChangeLog
+++ b/lisp/ChangeLog
@@ -1,3 +1,15 @@
+2012-08-14  Dmitry Gutov  <[email-masked]>
+
+	* progmodes/ruby-mode.el (ruby-percent-literal-beg-re): New constant.
+	(ruby-syntax-general-delimiters-goto-beg): Rename to
+	`ruby-syntax-enclosing-percent-literal', improve literal type check.
+	(ruby-syntax-propertize-general-delimiters): Rename to
+	`ruby-syntax-propertize-percent-literal', it's a shorter and more
+	popular term.  Adjust comments everywhere.
+	(ruby-syntax-propertize-percent-literal): Only propertize when not
+	inside a simple string or comment.  When the literal is unclosed,
+	leave the text after it unpropertized.
+
 2012-08-13  Andreas Schwab  <[email-masked]>
 
 	* files.el (hack-local-variables-filter): Remove useless eval.
diff --git a/lisp/progmodes/ruby-mode.el b/lisp/progmodes/ruby-mode.el
index 744dd43..42e1ac7 100644
--- a/lisp/progmodes/ruby-mode.el
+++ b/lisp/progmodes/ruby-mode.el
@@ -800,7 +800,7 @@ and `\\' when preceded by `?'."
 ;;                           (not (or (eolp) (looking-at "#")
 ;;                                    (and (eq (car (nth 1 state)) ?{)
 ;;                                         (looking-at "|"))))))
-                       ;; Not a regexp or general delimited literal.
+                       ;; Not a regexp or percent literal.
                        (null (nth 0 (ruby-parse-region (or begin parse-start)
                                                        (point))))
                        (or (not (eq ?| (char-after (point))))
@@ -1169,17 +1169,22 @@ See `add-log-current-defun-function'."
       (ruby-do-end-to-brace)))
 
 (declare-function ruby-syntax-propertize-heredoc "ruby-mode" (limit))
-(declare-function ruby-syntax-general-delimiters-goto-beg "ruby-mode" ())
-(declare-function ruby-syntax-propertize-general-delimiters "ruby-mode" (limit))
+(declare-function ruby-syntax-enclosing-percent-literal "ruby-mode" (limit))
+(declare-function ruby-syntax-propertize-percent-literal "ruby-mode" (limit))
 
 (if (eval-when-compile (fboundp #'syntax-propertize-rules))
     ;; New code that works independently from font-lock.
     (progn
+      (eval-and-compile
+        (defconst ruby-percent-literal-beg-re
+          "\\(%\\)[qQrswWx]?\\([[:punct:]]\\)"
+          "Regexp to match the beginning of percent literal."))
+
       (defun ruby-syntax-propertize-function (start end)
         "Syntactic keywords for Ruby mode.  See `syntax-propertize-function'."
         (goto-char start)
         (ruby-syntax-propertize-heredoc end)
-        (ruby-syntax-general-delimiters-goto-beg)
+        (ruby-syntax-enclosing-percent-literal end)
         (funcall
          (syntax-propertize-rules
           ;; #{ }, #$hoge, #@foo are not comments.
@@ -1222,8 +1227,8 @@ See `add-log-current-defun-function'."
                                    'syntax-table (string-to-syntax "\""))
                 (ruby-syntax-propertize-heredoc end))))
           ;; Handle percent literals: %w(), %q{}, etc.
-          ("\\(?:^\\|[[ \t\n<+(,=]\\)\\(%\\)[qQrswWx]?\\([[:punct:]]\\)"
-           (1 (prog1 "|" (ruby-syntax-propertize-general-delimiters end)))))
+          ((concat "\\(?:^\\|[[ \t\n<+(,=]\\)" ruby-percent-literal-beg-re)
+           (1 (prog1 "|" (ruby-syntax-propertize-percent-literal end)))))
          (point) end))
 
       (defun ruby-syntax-propertize-heredoc (limit)
@@ -1251,40 +1256,46 @@ See `add-log-current-defun-function'."
               ;; inf-loop.
               (if (< (point) start) (goto-char start))))))
 
-      (defun ruby-syntax-general-delimiters-goto-beg ()
-        (let ((state (syntax-ppss)))
-          ;; Move to the start of the literal, in case it's multiline.
-          ;; TODO: determine the literal type more reliably here?
+      (defun ruby-syntax-enclosing-percent-literal (limit)
+        (let ((state (syntax-ppss))
+              (start (point)))
+          ;; When already inside percent literal, re-propertize it.
           (when (eq t (nth 3 state))
             (goto-char (nth 8 state))
-            (beginning-of-line))))
+            (when (looking-at ruby-percent-literal-beg-re)
+              (ruby-syntax-propertize-percent-literal limit))
+            (when (< (point) start) (goto-char start)))))
 
-      (defun ruby-syntax-propertize-general-delimiters (limit)
+      (defun ruby-syntax-propertize-percent-literal (limit)
         (goto-char (match-beginning 2))
-        (let* ((op (char-after))
-               (ops (char-to-string op))
-               (cl (or (cdr (aref (syntax-table) op))
-                       (cdr (assoc op '((?< . ?>))))))
-               parse-sexp-lookup-properties)
-          (ignore-errors
-            (if cl
-                (progn  ; Paired delimiters.
-                  ;; Delimiter pairs of the same kind can be nested
-                  ;; inside the literal, as long as they are balanced.
-                  ;; Create syntax table that ignores other characters.
-                  (with-syntax-table (make-char-table 'syntax-table nil)
-                    (modify-syntax-entry op (concat "(" (char-to-string cl)))
-                    (modify-syntax-entry cl (concat ")" ops))
-                    (modify-syntax-entry ?\\ "\\")
-                    (save-restriction
-                      (narrow-to-region (point) limit)
-                      (forward-list))))  ; skip to the paired character
-              ;; Single character delimiter.
-              (re-search-forward (concat "[^\\]\\(?:\\\\\\\\\\)*"
-                                         (regexp-quote ops)) limit nil))
-            ;; If we reached here, the closing delimiter was found.
-            (put-text-property (1- (point)) (point)
-                               'syntax-table (string-to-syntax "|")))))
+        ;; Not inside a simple string or comment.
+        (when (eq t (nth 3 (syntax-ppss)))
+          (let* ((op (char-after))
+                 (ops (char-to-string op))
+                 (cl (or (cdr (aref (syntax-table) op))
+                         (cdr (assoc op '((?< . ?>))))))
+                 parse-sexp-lookup-properties)
+            (condition-case nil
+                (progn
+                  (if cl ; Paired delimiters.
+                      ;; Delimiter pairs of the same kind can be nested
+                      ;; inside the literal, as long as they are balanced.
+                      ;; Create syntax table that ignores other characters.
+                      (with-syntax-table (make-char-table 'syntax-table nil)
+                        (modify-syntax-entry op (concat "(" (char-to-string cl)))
+                        (modify-syntax-entry cl (concat ")" ops))
+                        (modify-syntax-entry ?\\ "\\")
+                        (save-restriction
+                          (narrow-to-region (point) limit)
+                          (forward-list))) ; skip to the paired character
+                    ;; Single character delimiter.
+                    (re-search-forward (concat "[^\\]\\(?:\\\\\\\\\\)*"
+                                               (regexp-quote ops)) limit nil))
+                  ;; Found the closing delimiter.
+                  (put-text-property (1- (point)) (point) 'syntax-table
+                                     (string-to-syntax "|")))
+              ;; Unclosed literal, leave the following text unpropertized.
+              ((scan-error search-failed) (goto-char limit))))))
       )
 
   ;; For Emacsen where syntax-propertize-rules is not (yet) available,
@@ -1329,7 +1340,7 @@ This should only be called after matching against `ruby-here-doc-end-re'."
      (4 (7 . ?/))
      (6 (7 . ?/)))
     ("^=en\\(d\\)\\_>" 1 "!")
-    ;; General delimited string.
+    ;; Percent literal.
     ("\\(^\\|[[ \t\n<+(,=]\\)\\(%[xrqQwW]?\\([^<[{(a-zA-Z0-9 \n]\\)[^\n\\\\]*\\(\\\\.[^\n\\\\]*\\)*\\(\\3\\)\\)"
      (3 "\"")
      (5 "\""))
diff --git a/test/ChangeLog b/test/ChangeLog
index a0163b5..9dbca3d 100644
--- a/test/ChangeLog
+++ b/test/ChangeLog
@@ -1,3 +1,7 @@
+2012-08-14  Dmitry Gutov  <[email-masked]>
+
+	* indent/ruby.rb: New examples.
+
 2012-08-12  Dmitry Gutov  <[email-masked]>
 
 	* automated/ruby-mode-tests.el (ruby-move-to-block-stops-at-opening)
diff --git a/test/indent/ruby.rb b/test/indent/ruby.rb
index c4a747a..7a9d123 100644
--- a/test/indent/ruby.rb
+++ b/test/indent/ruby.rb
@@ -7,6 +7,13 @@ c = %w(foo
  baz)
 d = %!hello!
 
+# Don't propertize percent literals inside strings.
+"(%s, %s)" % [123, 456]
+
+# Nor inside comments.
+x = # "tot %q/to"; =
+y = 2 / 3
+
 # A "do" after a slash means that slash is not a division, but it doesn't imply
 # it's a regexp-ender, since it can be a regexp-starter instead!
 x = toto / foo; if /do bar/ then
diff --git a/lisp/ChangeLog b/lisp/ChangeLog
index 4f52796..f37c346 100644
--- a/lisp/ChangeLog
+++ b/lisp/ChangeLog
@@ -9,6 +9,9 @@
 	(ruby-syntax-propertize-percent-literal): Only propertize when not
 	inside a simple string or comment.  When the literal is unclosed,
 	leave the text after it unpropertized.
+	(ruby-syntax-methods-before-regexp): New constant.
+	(ruby-syntax-propertize-function): Use it to recognize regexps.
+	Don't look at the text after regexp, just use the whitelist.
 
 2012-08-13  Andreas Schwab  <[email-masked]>
 
diff --git a/lisp/progmodes/ruby-mode.el b/lisp/progmodes/ruby-mode.el
index 42e1ac7..457c7fe 100644
--- a/lisp/progmodes/ruby-mode.el
+++ b/lisp/progmodes/ruby-mode.el
@@ -1178,7 +1178,13 @@ See `add-log-current-defun-function'."
       (eval-and-compile
         (defconst ruby-percent-literal-beg-re
           "\\(%\\)[qQrswWx]?\\([[:punct:]]\\)"
-          "Regexp to match the beginning of percent literal."))
+          "Regexp to match the beginning of percent literal.")
+
+        (defconst ruby-syntax-methods-before-regexp
+          '("gsub" "gsub!" "sub" "sub!" "scan" "split" "split!" "index" "match"
+            "assert_match" "Given" "Then" "When")
+          "Methods that can take regexp as the first argument.
+It will be properly highlighted even when the call omits parens."))
 
       (defun ruby-syntax-propertize-function (start end)
         "Syntactic keywords for Ruby mode.  See `syntax-propertize-function'."
@@ -1196,28 +1202,23 @@ See `add-log-current-defun-function'."
                         ;; Not within a string.
                         (nth 3 (syntax-ppss (match-beginning 0))))
                 (string-to-syntax "\\"))))
-          ;; Regexps: regexps are distinguished from division either because
-          ;; of the keyword/symbol before them, or because of the code
-          ;; following them.
+          ;; Regexps: regexps are distinguished from division because
+          ;; of the keyword, symbol, or method name before them.
           ((concat
             ;; Special tokens that can't be followed by a division operator.
-            "\\(?:\\(^\\|[[=(,~?:;<>]\\|\\(?:^\\|\\s \\)"
+            "\\(^\\|[[=(,~?:;<>]"
+            ;; Control flow keywords and operators following bol or whitespace.
+            "\\|\\(?:^\\|\\s \\)"
             (regexp-opt '("if" "elsif" "unless" "while" "until" "when" "and"
-                          "or" "&&" "||"
-                          "gsub" "gsub!" "sub" "sub!" "scan" "split" "split!"))
-            "\\)\\s *\\)?"
+                          "or" "not" "&&" "||"))
+            ;; Method name from the list.
+            "\\|\\_<"
+            (regexp-opt ruby-syntax-methods-before-regexp)
+            "\\)\\s *"
             ;; The regular expression itself.
-            "\\(/\\)[^/\n\\\\]*\\(?:\\\\.[^/\n\\\\]*\\)*\\(/\\)"
-            ;; Special code that cannot follow a division operator.
-            ;; FIXME: Just because the second slash of "/foo/ do bar" can't
-            ;; be a division, doesn't mean it can't *start* a regexp, as in
-            ;; "x = toto/foo; if /do bar/".
-            "\\([imxo]*\\s *\\(?:,\\|\\_<do\\_>\\)\\)?")
-           (2 (when (or (match-beginning 1) (match-beginning 4))
-                (string-to-syntax "\"/")))
-           (3 (if (or (match-beginning 1) (match-beginning 4))
-                  (string-to-syntax "\"/")
-                (goto-char (match-end 2)))))
+            "\\(/\\)[^/\n\\\\]*\\(?:\\\\.[^/\n\\\\]*\\)*\\(/\\)")
+           (2 (string-to-syntax "\"/"))
+           (3 (string-to-syntax "\"/")))
           ("^=en\\(d\\)\\_>" (1 "!"))
           ("^\\(=\\)begin\\_>" (1 "!"))
           ;; Handle here documents.
diff --git a/test/ChangeLog b/test/ChangeLog
index 9dbca3d..f1bf458 100644
--- a/test/ChangeLog
+++ b/test/ChangeLog
@@ -1,6 +1,6 @@
 2012-08-14  Dmitry Gutov  <[email-masked]>
 
-	* indent/ruby.rb: New examples.
+	* indent/ruby.rb: Rearrange examples, add new ones.
 
 2012-08-12  Dmitry Gutov  <[email-masked]>
 
diff --git a/test/indent/ruby.rb b/test/indent/ruby.rb
index 7a9d123..4f2e9e6 100644
--- a/test/indent/ruby.rb
+++ b/test/indent/ruby.rb
@@ -1,24 +1,25 @@
-# Don't mis-match "sub" at the end of words.
-a = asub / aslb + bsub / bslb;
-
+# Percent literals.
 b = %Q{This is a "string"}
-c = %w(foo
+c = %w!foo
  bar
- baz)
-d = %!hello!
+ baz!
+d = %(hello (nested) world)
 
 # Don't propertize percent literals inside strings.
 "(%s, %s)" % [123, 456]
 
-# Nor inside comments.
+# Or inside comments.
 x = # "tot %q/to"; =
 y = 2 / 3
 
-# A "do" after a slash means that slash is not a division, but it doesn't imply
-# it's a regexp-ender, since it can be a regexp-starter instead!
-x = toto / foo; if /do bar/ then
-                  toto = 1
-                end
+# Regexp after whitelisted method.
+"abc".sub /b/, 'd'
+
+# Don't mis-match "sub" at the end of words.
+a = asub / aslb + bsub / bslb;
+
+# Highlight the regexp after "if".
+x = toto / foo if /do bar/ =~ "dobar"
 
 # Some Cucumber code:
 Given /toto/ do