1 ;;; syntax.el --- helper functions to find syntactic context
3 ;; Copyright (C) 2000, 2001, 2002, 2003, 2004,
4 ;; 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
9 ;; This file is part of GNU Emacs.
11 ;; GNU Emacs is free software: you can redistribute it and/or modify
12 ;; it under the terms of the GNU General Public License as published by
13 ;; the Free Software Foundation, either version 3 of the License, or
14 ;; (at your option) any later version.
16 ;; GNU Emacs is distributed in the hope that it will be useful,
17 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
18 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 ;; GNU General Public License for more details.
21 ;; You should have received a copy of the GNU General Public License
22 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
26 ;; The main exported function is `syntax-ppss'. You might also need
27 ;; to call `syntax-ppss-flush-cache' or to add it to
28 ;; before-change-functions'(although this is automatically done by
29 ;; syntax-ppss when needed, but that might fail if syntax-ppss is
30 ;; called in a context where before-change-functions is temporarily
35 ;; - do something about the case where the syntax-table is changed.
36 ;; This typically happens with tex-mode and its `$' operator.
37 ;; - new functions `syntax-state', ... to replace uses of parse-partial-state
38 ;; with something higher-level (similar to syntax-ppss-context).
39 ;; - interaction with mmm-mode.
43 ;; Note: PPSS stands for `parse-partial-sexp state'
45 (eval-when-compile (require 'cl
))
47 (defvar font-lock-beginning-of-syntax-function
)
49 ;;; Applying syntax-table properties where needed.
51 (defvar syntax-propertize-function nil
52 ;; Rather than a -functions hook, this is a -function because it's easier
53 ;; to do a single scan than several scans: with multiple scans, one cannot
54 ;; assume that the text before point has been propertized, so syntax-ppss
55 ;; gives unreliable results (and stores them in its cache to boot, so we'd
56 ;; have to flush that cache between each function, and we couldn't use
57 ;; syntax-ppss-flush-cache since that would not only flush the cache but also
58 ;; reset syntax-propertize--done which should not be done in this case).
59 "Mode-specific function to apply the syntax-table properties.
60 Called with 2 arguments: START and END.
61 This function can call `syntax-ppss' on any position before END, but it
62 should not call `syntax-ppss-flush-cache', which means that it should not
63 call `syntax-ppss' on some position and later modify the buffer on some
66 (defvar syntax-propertize-chunk-size
500)
68 (defvar syntax-propertize-extend-region-functions
69 '(syntax-propertize-wholelines)
70 "Special hook run just before proceeding to propertize a region.
71 This is used to allow major modes to help `syntax-propertize' find safe buffer
72 positions as beginning and end of the propertized region. Its most common use
73 is to solve the problem of /identification/ of multiline elements by providing
74 a function that tries to find such elements and move the boundaries such that
75 they do not fall in the middle of one.
76 Each function is called with two arguments (START and END) and it should return
77 either a cons (NEW-START . NEW-END) or nil if no adjustment should be made.
78 These functions are run in turn repeatedly until they all return nil.
79 Put first the functions more likely to cause a change and cheaper to compute.")
80 ;; Mark it as a special hook which doesn't use any global setting
81 ;; (i.e. doesn't obey the element t in the buffer-local value).
82 (make-variable-buffer-local 'syntax-propertize-extend-region-functions
)
84 (defun syntax-propertize-wholelines (start end
)
86 (cons (line-beginning-position)
87 (progn (goto-char end
)
88 (if (bolp) (point) (line-beginning-position 2)))))
90 (defun syntax-propertize-multiline (beg end
)
91 "Let `syntax-propertize' pay attention to the syntax-multiline property."
92 (when (and (> beg
(point-min))
93 (get-text-property (1- beg
) 'syntax-multiline
))
94 (setq beg
(or (previous-single-property-change beg
'syntax-multiline
)
97 (when (get-text-property end
'font-lock-multiline
)
98 (setq end
(or (text-property-any end
(point-max)
99 'syntax-multiline nil
)
103 (defvar syntax-propertize--done -
1
104 "Position upto which syntax-table properties have been set.")
105 (make-variable-buffer-local 'syntax-propertize--done
)
107 (defun syntax-propertize--shift-groups (re n
)
108 (replace-regexp-in-string
109 "\\\\(\\?\\([0-9]+\\):"
112 (number-to-string (+ n
(string-to-number (match-string 1 s
))))
116 (defmacro syntax-propertize-precompile-rules
(&rest rules
)
117 "Return a precompiled form of RULES to pass to `syntax-propertize-rules'.
118 The arg RULES can be of the same form as in `syntax-propertize-rules'.
119 The return value is an object that can be passed as a rule to
120 `syntax-propertize-rules'.
121 I.e. this is useful only when you want to share rules among several
122 syntax-propertize-functions."
123 (declare (debug syntax-propertize-rules
))
124 ;; Precompile? Yeah, right!
125 ;; Seriously, tho, this is a macro for 2 reasons:
126 ;; - we could indeed do some pre-compilation at some point in the future,
127 ;; e.g. fi/when we switch to a DFA-based implementation of
128 ;; syntax-propertize-rules.
129 ;; - this lets Edebug properly annotate the expressions inside RULES.
132 (defmacro syntax-propertize-rules
(&rest rules
)
133 "Make a function that applies RULES for use in `syntax-propertize-function'.
134 The function will scan the buffer, applying the rules where they match.
135 The buffer is scanned a single time, like \"lex\" would, rather than once
138 Each RULE can be a symbol, in which case that symbol's value should be,
139 at macro-expansion time, a precompiled set of rules, as returned
140 by `syntax-propertize-precompile-rules'.
142 Otherwise, RULE should have the form (REGEXP HIGHLIGHT1 ... HIGHLIGHTn), where
143 REGEXP is an expression (evaluated at time of macro-expansion) that returns
144 a regexp, and where HIGHLIGHTs have the form (NUMBER SYNTAX) which means to
145 apply the property SYNTAX to the chars matched by the subgroup NUMBER
146 of the regular expression, if NUMBER did match.
147 SYNTAX is an expression that returns a value to apply as `syntax-table'
148 property. Some expressions are handled specially:
149 - if SYNTAX is a string, then it is converted with `string-to-syntax';
150 - if SYNTAX has the form (prog1 EXP . EXPS) then the value returned by EXP
151 will be applied to the buffer before running EXPS and if EXP is a string it
152 is also converted with `string-to-syntax'.
153 The SYNTAX expression is responsible to save the `match-data' if needed
154 for subsequent HIGHLIGHTs.
155 Also SYNTAX is free to move point, in which case RULES may not be applied to
156 some parts of the text or may be applied several times to other parts.
158 Note: back-references in REGEXPs do not work."
159 (declare (debug (&rest
&or symbolp
;FIXME: edebug this eval step.
162 [&or stringp
;FIXME: Use &wrap
163 ("prog1" [&or stringp def-form
] def-body
)
165 (let ((newrules nil
))
167 (if (symbolp (car rules
))
168 (setq rules
(append (symbol-value (pop rules
)) rules
))
169 (push (pop rules
) newrules
)))
170 (setq rules
(nreverse newrules
)))
173 ;; We'd like to use a real DFA-based lexer, usually, but since Emacs
174 ;; doesn't have one yet, we fallback on building one large regexp
175 ;; and use groups to determine which branch of the regexp matched.
179 (let* ((orig-re (eval (car rule
)))
181 (when (and (assq 0 rule
) (cdr rules
))
182 ;; If there's more than 1 rule, and the rule want to apply
183 ;; highlight to match 0, create an extra group to be able to
184 ;; tell when *this* match 0 has succeeded.
186 (setq re
(concat "\\(" re
"\\)")))
187 (setq re
(syntax-propertize--shift-groups re offset
))
191 ((assq 0 rule
) (if (zerop offset
) t
192 `(match-beginning ,offset
)))
194 `(match-beginning ,(+ offset
(car (cadr rule
)))))
198 `(match-beginning ,(+ offset
(car case
))))
202 ;; If some of the subgroup rules include Elisp code, then we
203 ;; need to set the match-data so it's consistent with what the
204 ;; code expects. If not, then we can simply use shifted
205 ;; offset in our own code.
206 (unless (zerop offset
)
207 (dolist (case (cdr rule
))
208 (unless (stringp (cadr case
))
211 (push `(let ((md (match-data 'ints
)))
212 ;; Keep match 0 as is, but shift everything else.
213 (setcdr (cdr md
) (nthcdr ,(* (1+ offset
) 2) md
))
217 ;; Now construct the code for each subgroup rules.
218 (dolist (case (cdr rule
))
219 (assert (null (cddr case
)))
220 (let* ((gn (+ offset
(car case
)))
221 (action (nth 1 case
))
226 (match-beginning ,gn
) (match-end ,gn
)
228 ',(string-to-syntax action
))))
229 ((eq (car-safe action
) 'ignore
)
231 ((eq (car-safe action
) 'prog1
)
232 (if (stringp (nth 1 action
))
234 (match-beginning ,gn
) (match-end ,gn
)
236 ',(string-to-syntax (nth 1 action
)))
238 `((let ((mb (match-beginning ,gn
))
240 (syntax ,(nth 1 action
)))
243 mb me
'syntax-table syntax
))
244 ,@(nthcdr 2 action
)))))
246 `((let ((mb (match-beginning ,gn
))
251 mb me
'syntax-table syntax
))))))))
253 (if (or (not (cddr rule
)) (zerop gn
))
254 (setq code
(nconc (nreverse thiscode
) code
))
255 (push `(if (match-beginning ,gn
)
256 ;; Try and generate clean code with no
258 ,(if (null (cdr thiscode
))
260 `(progn ,@thiscode
)))
262 (push (cons condition
(nreverse code
))
264 (incf offset
(regexp-opt-depth orig-re
))
270 (while (and (< (point) end
)
271 (re-search-forward ,re end t
))
272 (cond ,@(nreverse branches
))))))
274 (defun syntax-propertize-via-font-lock (keywords)
275 "Propertize for syntax in START..END using font-lock syntax.
276 KEYWORDS obeys the format used in `font-lock-syntactic-keywords'.
277 The return value is a function suitable for `syntax-propertize-function'."
278 (lexical-let ((keywords keywords
))
281 (let ((font-lock-syntactic-keywords keywords
))
282 (font-lock-fontify-syntactic-keywords-region start end
)
283 ;; In case it was eval'd/compiled.
284 (setq keywords font-lock-syntactic-keywords
))))))
286 (defun syntax-propertize (pos)
287 "Ensure that syntax-table properties are set upto POS."
288 (when (and syntax-propertize-function
289 (< syntax-propertize--done pos
))
290 ;; (message "Needs to syntax-propertize from %s to %s"
291 ;; syntax-propertize--done pos)
292 (set (make-local-variable 'parse-sexp-lookup-properties
) t
)
294 (with-silent-modifications
295 (let* ((start (max syntax-propertize--done
(point-min)))
298 (+ start syntax-propertize-chunk-size
))))
299 (funs syntax-propertize-extend-region-functions
))
301 (let ((new (funcall (pop funs
) start end
)))
303 (and (>= (car new
) start
) (<= (cdr new
) end
)))
305 (setq start
(car new
))
307 ;; If there's been a change, we should go through the
308 ;; list again since this new position may
309 ;; warrant a different answer from one of the funs we've
312 (cdr syntax-propertize-extend-region-functions
))
313 (setq funs syntax-propertize-extend-region-functions
)))))
314 ;; Move the limit before calling the function, so the function
315 ;; can use syntax-ppss.
316 (setq syntax-propertize--done end
)
317 ;; (message "syntax-propertizing from %s to %s" start end)
318 (remove-text-properties start end
319 '(syntax-table nil syntax-multiline nil
))
320 (funcall syntax-propertize-function start end
))))))
322 ;;; Incrementally compute and memoize parser state.
324 (defsubst syntax-ppss-depth
(ppss)
327 (defun syntax-ppss-toplevel-pos (ppss)
328 "Get the latest syntactically outermost position found in a syntactic scan.
329 PPSS is a scan state, as returned by `parse-partial-sexp' or `syntax-ppss'.
330 An \"outermost position\" means one that it is outside of any syntactic entity:
331 outside of any parentheses, comments, or strings encountered in the scan.
332 If no such position is recorded in PPSS (because the end of the scan was
333 itself at the outermost level), return nil."
334 ;; BEWARE! We rely on the undocumented 9th field. The 9th field currently
335 ;; contains the list of positions of the enclosing open-parens.
336 ;; I.e. those positions are outside of any string/comment and the first of
337 ;; those is outside of any paren (i.e. corresponds to a nil ppss).
338 ;; If this list is empty but we are in a string or comment, then the 8th
339 ;; field contains a similar "toplevel" position.
340 (or (car (nth 9 ppss
))
343 (defsubst syntax-ppss-context
(ppss)
345 ((nth 3 ppss
) 'string
)
346 ((nth 4 ppss
) 'comment
)
349 (defvar syntax-ppss-max-span
20000
350 "Threshold below which cache info is deemed unnecessary.
351 We try to make sure that cache entries are at least this far apart
352 from each other, to avoid keeping too much useless info.")
354 (defvar syntax-begin-function nil
355 "Function to move back outside of any comment/string/paren.
356 This function should move the cursor back to some syntactically safe
357 point (where the PPSS is equivalent to nil).")
359 (defvar syntax-ppss-cache nil
360 "List of (POS . PPSS) pairs, in decreasing POS order.")
361 (make-variable-buffer-local 'syntax-ppss-cache
)
362 (defvar syntax-ppss-last nil
363 "Cache of (LAST-POS . LAST-PPSS).")
364 (make-variable-buffer-local 'syntax-ppss-last
)
366 (defalias 'syntax-ppss-after-change-function
'syntax-ppss-flush-cache
)
367 (defun syntax-ppss-flush-cache (beg &rest ignored
)
368 "Flush the cache of `syntax-ppss' starting at position BEG."
369 ;; Set syntax-propertize to refontify anything past beg.
370 (setq syntax-propertize--done
(min beg syntax-propertize--done
))
371 ;; Flush invalid cache entries.
372 (while (and syntax-ppss-cache
(> (caar syntax-ppss-cache
) beg
))
373 (setq syntax-ppss-cache
(cdr syntax-ppss-cache
)))
374 ;; Throw away `last' value if made invalid.
375 (when (< beg
(or (car syntax-ppss-last
) 0))
376 ;; If syntax-begin-function jumped to BEG, then the old state at BEG can
377 ;; depend on the text after BEG (which is presumably changed). So if
378 ;; BEG=(car (nth 10 syntax-ppss-last)) don't reuse that data because the
379 ;; assumed nil state at BEG may not be valid any more.
380 (if (<= beg
(or (syntax-ppss-toplevel-pos (cdr syntax-ppss-last
))
381 (nth 3 syntax-ppss-last
)
383 (setq syntax-ppss-last nil
)
384 (setcar syntax-ppss-last nil
)))
385 ;; Unregister if there's no cache left. Sadly this doesn't work
386 ;; because `before-change-functions' is temporarily bound to nil here.
387 ;; (unless syntax-ppss-cache
388 ;; (remove-hook 'before-change-functions 'syntax-ppss-flush-cache t))
391 (defvar syntax-ppss-stats
392 [(0 .
0.0) (0 .
0.0) (0 .
0.0) (0 .
0.0) (0 .
0.0) (1 .
2500.0)])
393 (defun syntax-ppss-stats ()
396 (cons (car x
) (truncate (/ (cdr x
) (car x
))))
400 (defun syntax-ppss (&optional pos
)
401 "Parse-Partial-Sexp State at POS, defaulting to point.
402 The returned value is the same as `parse-partial-sexp' except that
403 the 2nd and 6th values of the returned state cannot be relied upon.
404 Point is at POS when this function returns."
406 (unless pos
(setq pos
(point)))
407 (syntax-propertize pos
)
409 (let ((old-ppss (cdr syntax-ppss-last
))
410 (old-pos (car syntax-ppss-last
))
412 (pt-min (point-min)))
413 (if (and old-pos
(> old-pos pos
)) (setq old-pos nil
))
414 ;; Use the OLD-POS if usable and close. Don't update the `last' cache.
416 (if (and old-pos
(< (- pos old-pos
)
417 ;; The time to use syntax-begin-function and
418 ;; find PPSS is assumed to be about 2 * distance.
419 (* 2 (/ (cdr (aref syntax-ppss-stats
5))
420 (1+ (car (aref syntax-ppss-stats
5)))))))
422 (incf (car (aref syntax-ppss-stats
0)))
423 (incf (cdr (aref syntax-ppss-stats
0)) (- pos old-pos
))
424 (parse-partial-sexp old-pos pos nil nil old-ppss
))
427 ;; Use OLD-PPSS if possible and close enough.
428 ((and (not old-pos
) old-ppss
429 ;; If `pt-min' is too far from `pos', we could try to use
430 ;; other positions in (nth 9 old-ppss), but that doesn't
431 ;; seem to happen in practice and it would complicate this
432 ;; code (and the before-change-function code even more).
433 ;; But maybe it would be useful in "degenerate" cases such
434 ;; as when the whole file is wrapped in a set
436 (setq pt-min
(or (syntax-ppss-toplevel-pos old-ppss
)
438 (<= pt-min pos
) (< (- pos pt-min
) syntax-ppss-max-span
))
439 (incf (car (aref syntax-ppss-stats
1)))
440 (incf (cdr (aref syntax-ppss-stats
1)) (- pos pt-min
))
441 (setq ppss
(parse-partial-sexp pt-min pos
)))
442 ;; The OLD-* data can't be used. Consult the cache.
444 (let ((cache-pred nil
)
445 (cache syntax-ppss-cache
)
447 ;; I differentiate between PT-MIN and PT-BEST because
448 ;; I feel like it might be important to ensure that the
449 ;; cache is only filled with 100% sure data (whereas
450 ;; syntax-begin-function might return incorrect data).
451 ;; Maybe that's just stupid.
452 (pt-best (point-min))
454 ;; look for a usable cache entry.
455 (while (and cache
(< pos
(caar cache
)))
456 (setq cache-pred cache
)
457 (setq cache
(cdr cache
)))
458 (if cache
(setq pt-min
(caar cache
) ppss
(cdar cache
)))
460 ;; Setup the before-change function if necessary.
461 (unless (or syntax-ppss-cache syntax-ppss-last
)
462 (add-hook 'before-change-functions
463 'syntax-ppss-flush-cache t t
))
465 ;; Use the best of OLD-POS and CACHE.
466 (if (or (not old-pos
) (< old-pos pt-min
))
467 (setq pt-best pt-min ppss-best ppss
)
468 (incf (car (aref syntax-ppss-stats
4)))
469 (incf (cdr (aref syntax-ppss-stats
4)) (- pos old-pos
))
470 (setq pt-best old-pos ppss-best old-ppss
))
472 ;; Use the `syntax-begin-function' if available.
473 ;; We could try using that function earlier, but:
474 ;; - The result might not be 100% reliable, so it's better to use
475 ;; the cache if available.
476 ;; - The function might be slow.
477 ;; - If this function almost always finds a safe nearby spot,
478 ;; the cache won't be populated, so consulting it is cheap.
479 (when (and (not syntax-begin-function
)
480 (boundp 'font-lock-beginning-of-syntax-function
)
481 font-lock-beginning-of-syntax-function
)
482 (set (make-local-variable 'syntax-begin-function
)
483 font-lock-beginning-of-syntax-function
))
484 (when (and syntax-begin-function
485 (progn (goto-char pos
)
486 (funcall syntax-begin-function
)
487 ;; Make sure it's better.
489 ;; Simple sanity checks.
490 (< (point) pos
) ; backward-paragraph can fail here.
491 (not (memq (get-text-property (point) 'face
)
492 '(font-lock-string-face font-lock-doc-face
493 font-lock-comment-face
))))
494 (incf (car (aref syntax-ppss-stats
5)))
495 (incf (cdr (aref syntax-ppss-stats
5)) (- pos
(point)))
496 (setq pt-best
(point) ppss-best nil
))
499 ;; Quick case when we found a nearby pos.
500 ((< (- pos pt-best
) syntax-ppss-max-span
)
501 (incf (car (aref syntax-ppss-stats
2)))
502 (incf (cdr (aref syntax-ppss-stats
2)) (- pos pt-best
))
503 (setq ppss
(parse-partial-sexp pt-best pos nil nil ppss-best
)))
504 ;; Slow case: compute the state from some known position and
505 ;; populate the cache so we won't need to do it again soon.
507 (incf (car (aref syntax-ppss-stats
3)))
508 (incf (cdr (aref syntax-ppss-stats
3)) (- pos pt-min
))
510 ;; If `pt-min' is too far, add a few intermediate entries.
511 (while (> (- pos pt-min
) (* 2 syntax-ppss-max-span
))
512 (setq ppss
(parse-partial-sexp
513 pt-min
(setq pt-min
(/ (+ pt-min pos
) 2))
515 (let ((pair (cons pt-min ppss
)))
517 (push pair
(cdr cache-pred
))
518 (push pair syntax-ppss-cache
))))
520 ;; Compute the actual return value.
521 (setq ppss
(parse-partial-sexp pt-min pos nil nil ppss
))
524 ;; (let ((real-ppss (parse-partial-sexp (point-min) pos)))
525 ;; (setcar (last ppss 4) 0)
526 ;; (setcar (last real-ppss 4) 0)
527 ;; (setcar (last ppss 8) nil)
528 ;; (setcar (last real-ppss 8) nil)
529 ;; (unless (equal ppss real-ppss)
530 ;; (message "!!Syntax: %s != %s" ppss real-ppss)
531 ;; (setq ppss real-ppss)))
533 ;; Store it in the cache.
534 (let ((pair (cons pos ppss
)))
536 (if (> (- (caar cache-pred
) pos
) syntax-ppss-max-span
)
537 (push pair
(cdr cache-pred
))
538 (setcar cache-pred pair
))
539 (if (or (null syntax-ppss-cache
)
540 (> (- (caar syntax-ppss-cache
) pos
)
541 syntax-ppss-max-span
))
542 (push pair syntax-ppss-cache
)
543 (setcar syntax-ppss-cache pair
)))))))))
545 (setq syntax-ppss-last
(cons pos ppss
))
548 ;; If the buffer is more narrowed than when we built the cache,
549 ;; we may end up calling parse-partial-sexp with a position before
550 ;; point-min. In that case, just parse from point-min assuming
552 (parse-partial-sexp (point-min) pos
)))))
554 ;; Debugging functions
556 (defun syntax-ppss-debug ()
559 (dolist (x (append syntax-ppss-cache
(list (cons (point-min) nil
))))
560 (when pt
(push (- pt
(car x
)) min-diffs
))
564 ;; XEmacs compatibility functions
566 ;; (defun buffer-syntactic-context (&optional buffer)
567 ;; "Syntactic context at point in BUFFER.
568 ;; Either of `string', `comment' or `nil'.
569 ;; This is an XEmacs compatibility function."
570 ;; (with-current-buffer (or buffer (current-buffer))
571 ;; (syntax-ppss-context (syntax-ppss))))
573 ;; (defun buffer-syntactic-context-depth (&optional buffer)
574 ;; "Syntactic parenthesis depth at point in BUFFER.
575 ;; This is an XEmacs compatibility function."
576 ;; (with-current-buffer (or buffer (current-buffer))
577 ;; (syntax-ppss-depth (syntax-ppss))))
581 ;;; syntax.el ends here