1 ;;; syntax.el --- helper functions to find syntactic context
3 ;; Copyright (C) 2000, 2001, 2002, 2003, 2004,
4 ;; 2005, 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
9 ;; This file is part of GNU Emacs.
11 ;; GNU Emacs is free software: you can redistribute it and/or modify
12 ;; it under the terms of the GNU General Public License as published by
13 ;; the Free Software Foundation, either version 3 of the License, or
14 ;; (at your option) any later version.
16 ;; GNU Emacs is distributed in the hope that it will be useful,
17 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
18 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 ;; GNU General Public License for more details.
21 ;; You should have received a copy of the GNU General Public License
22 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
26 ;; The main exported function is `syntax-ppss'. You might also need
27 ;; to call `syntax-ppss-flush-cache' or to add it to
28 ;; before-change-functions'(although this is automatically done by
29 ;; syntax-ppss when needed, but that might fail if syntax-ppss is
30 ;; called in a context where before-change-functions is temporarily
35 ;; - do something about the case where the syntax-table is changed.
36 ;; This typically happens with tex-mode and its `$' operator.
37 ;; - new functions `syntax-state', ... to replace uses of parse-partial-state
38 ;; with something higher-level (similar to syntax-ppss-context).
39 ;; - interaction with mmm-mode.
43 ;; Note: PPSS stands for `parse-partial-sexp state'
45 (eval-when-compile (require 'cl
))
47 (defvar font-lock-beginning-of-syntax-function
)
49 ;;; Applying syntax-table properties where needed.
51 (defvar syntax-propertize-function nil
52 ;; Rather than a -functions hook, this is a -function because it's easier
53 ;; to do a single scan than several scans: with multiple scans, one cannot
54 ;; assume that the text before point has been propertized, so syntax-ppss
55 ;; gives unreliable results (and stores them in its cache to boot, so we'd
56 ;; have to flush that cache between each function, and we couldn't use
57 ;; syntax-ppss-flush-cache since that would not only flush the cache but also
58 ;; reset syntax-propertize--done which should not be done in this case).
59 "Mode-specific function to apply the syntax-table properties.
60 Called with 2 arguments: START and END.")
62 (defvar syntax-propertize-chunk-size
500)
64 (defvar syntax-propertize-extend-region-functions
65 '(syntax-propertize-wholelines)
66 "Special hook run just before proceeding to propertize a region.
67 This is used to allow major modes to help `syntax-propertize' find safe buffer
68 positions as beginning and end of the propertized region. Its most common use
69 is to solve the problem of /identification/ of multiline elements by providing
70 a function that tries to find such elements and move the boundaries such that
71 they do not fall in the middle of one.
72 Each function is called with two arguments (START and END) and it should return
73 either a cons (NEW-START . NEW-END) or nil if no adjustment should be made.
74 These functions are run in turn repeatedly until they all return nil.
75 Put first the functions more likely to cause a change and cheaper to compute.")
76 ;; Mark it as a special hook which doesn't use any global setting
77 ;; (i.e. doesn't obey the element t in the buffer-local value).
78 (make-variable-buffer-local 'syntax-propertize-extend-region-functions
)
80 (defun syntax-propertize-wholelines (start end
)
82 (cons (line-beginning-position)
83 (progn (goto-char end
)
84 (if (bolp) (point) (line-beginning-position 2)))))
86 (defun syntax-propertize-multiline (beg end
)
87 "Let `syntax-propertize' pay attention to the syntax-multiline property."
88 (when (and (> beg
(point-min))
89 (get-text-property (1- beg
) 'syntax-multiline
))
90 (setq beg
(or (previous-single-property-change beg
'syntax-multiline
)
93 (when (get-text-property end
'font-lock-multiline
)
94 (setq end
(or (text-property-any end
(point-max)
95 'syntax-multiline nil
)
99 (defvar syntax-propertize--done -
1
100 "Position upto which syntax-table properties have been set.")
101 (make-variable-buffer-local 'syntax-propertize--done
)
103 (defun syntax-propertize--shift-groups (re n
)
104 (replace-regexp-in-string
105 "\\\\(\\?\\([0-9]+\\):"
108 (number-to-string (+ n
(string-to-number (match-string 1 s
))))
112 (defmacro syntax-propertize-rules
(&rest rules
)
113 "Make a function that applies RULES for use in `syntax-propertize-function'.
114 The function will scan the buffer, applying the rules where they match.
115 The buffer is scanned a single time, like \"lex\" would, rather than once
118 Each rule has the form (REGEXP HIGHLIGHT1 ... HIGHLIGHTn), where REGEXP
119 is an expression (evaluated at time of macro-expansion) that returns a regexp,
120 and where HIGHLIGHTs have the form (NUMBER SYNTAX) which means to
121 apply the property SYNTAX to the chars matched by the subgroup NUMBER
122 of the regular expression, if NUMBER did match.
123 SYNTAX is an expression that returns a value to apply as `syntax-table'
124 property. Some expressions are handled specially:
125 - if SYNTAX is a string, then it is converted with `string-to-syntax';
126 - if SYNTAX has the form (prog1 EXP . EXPS) then the value returned by EXP
127 will be applied to the buffer before running EXPS and if EXP is a string it
128 is also converted with `string-to-syntax'.
129 The SYNTAX expression is responsible to save the `match-data' if needed
130 for subsequent HIGHLIGHTs.
131 Also SYNTAX is free to move point, in which case RULES may not be applied to
132 some parts of the text or may be applied several times to other parts.
134 Note: back-references in REGEXPs do not work."
135 (declare (debug (&rest
(form &rest
138 ("prog1" [&or stringp def-form
] def-body
)
142 ;; We'd like to use a real DFA-based lexer, usually, but since Emacs
143 ;; doesn't have one yet, we fallback on building one large regexp
144 ;; and use groups to determine which branch of the regexp matched.
148 (let ((re (eval (car rule
))))
149 (when (and (assq 0 rule
) (cdr rules
))
150 ;; If there's more than 1 rule, and the rule want to apply
151 ;; highlight to match 0, create an extra group to be able to
152 ;; tell when *this* match 0 has succeeded.
154 (setq re
(concat "\\(" re
"\\)")))
155 (setq re
(syntax-propertize--shift-groups re offset
))
159 ((assq 0 rule
) (if (zerop offset
) t
160 `(match-beginning ,offset
)))
162 `(match-beginning ,(+ offset
(car (cadr rule
)))))
166 `(match-beginning ,(+ offset
(car case
))))
170 ;; If some of the subgroup rules include Elisp code, then we
171 ;; need to set the match-data so it's consistent with what the
172 ;; code expects. If not, then we can simply use shifted
173 ;; offset in our own code.
174 (unless (zerop offset
)
175 (dolist (case (cdr rule
))
176 (unless (stringp (cadr case
))
179 (push `(let ((md (match-data 'ints
)))
180 ;; Keep match 0 as is, but shift everything else.
181 (setcdr (cdr md
) (nthcdr ,(* (1+ offset
) 2) md
))
185 ;; Now construct the code for each subgroup rules.
186 (dolist (case (cdr rule
))
187 (assert (null (cddr case
)))
188 (let* ((gn (+ offset
(car case
)))
189 (action (nth 1 case
))
194 (match-beginning ,gn
) (match-end ,gn
)
196 ',(string-to-syntax action
))))
197 ((eq (car-safe action
) 'ignore
)
199 ((eq (car-safe action
) 'prog1
)
200 (if (stringp (nth 1 action
))
202 (match-beginning ,gn
) (match-end ,gn
)
204 ',(string-to-syntax (nth 1 action
)))
206 `((let ((mb (match-beginning ,gn
))
208 (syntax ,(nth 1 action
)))
211 mb me
'syntax-table syntax
))
212 ,@(nthcdr 2 action
)))))
214 `((let ((mb (match-beginning ,gn
))
219 mb me
'syntax-table syntax
))))))))
221 (if (or (not (cddr rule
)) (zerop gn
))
222 (setq code
(nconc (nreverse thiscode
) code
))
223 (push `(if (match-beginning ,gn
)
224 ;; Try and generate clean code with no
226 ,(if (null (cdr thiscode
))
228 `(progn ,@thiscode
)))
230 (push (cons condition
(nreverse code
))
232 (incf offset
(regexp-opt-depth re
))
238 (while (and (< (point) end
)
239 (re-search-forward ,re end t
))
240 (cond ,@(nreverse branches
))))))
242 (defun syntax-propertize-via-font-lock (keywords)
243 "Propertize for syntax in START..END using font-lock syntax.
244 KEYWORDS obeys the format used in `font-lock-syntactic-keywords'.
245 The return value is a function suitable for `syntax-propertize-function'."
246 (lexical-let ((keywords keywords
))
249 (let ((font-lock-syntactic-keywords keywords
))
250 (font-lock-fontify-syntactic-keywords-region start end
)
251 ;; In case it was eval'd/compiled.
252 (setq keywords font-lock-syntactic-keywords
))))))
254 (defun syntax-propertize (pos)
255 "Ensure that syntax-table properties are set upto POS."
256 (when (and syntax-propertize-function
257 (< syntax-propertize--done pos
))
258 ;; (message "Needs to syntax-propertize from %s to %s"
259 ;; syntax-propertize--done pos)
260 (set (make-local-variable 'parse-sexp-lookup-properties
) t
)
262 (with-silent-modifications
263 (let* ((start (max syntax-propertize--done
(point-min)))
266 (+ start syntax-propertize-chunk-size
))))
267 (funs syntax-propertize-extend-region-functions
))
269 (let ((new (funcall (pop funs
) start end
)))
271 (and (>= (car new
) start
) (<= (cdr new
) end
)))
273 (setq start
(car new
))
275 ;; If there's been a change, we should go through the
276 ;; list again since this new position may
277 ;; warrant a different answer from one of the funs we've
280 (cdr syntax-propertize-extend-region-functions
))
281 (setq funs syntax-propertize-extend-region-functions
)))))
282 ;; Move the limit before calling the function, so the function
283 ;; can use syntax-ppss.
284 (setq syntax-propertize--done end
)
285 ;; (message "syntax-propertizing from %s to %s" start end)
286 (remove-text-properties start end
287 '(syntax-table nil syntax-multiline nil
))
288 (funcall syntax-propertize-function start end
))))))
290 ;;; Incrementally compute and memoize parser state.
292 (defsubst syntax-ppss-depth
(ppss)
295 (defun syntax-ppss-toplevel-pos (ppss)
296 "Get the latest syntactically outermost position found in a syntactic scan.
297 PPSS is a scan state, as returned by `parse-partial-sexp' or `syntax-ppss'.
298 An \"outermost position\" means one that it is outside of any syntactic entity:
299 outside of any parentheses, comments, or strings encountered in the scan.
300 If no such position is recorded in PPSS (because the end of the scan was
301 itself at the outermost level), return nil."
302 ;; BEWARE! We rely on the undocumented 9th field. The 9th field currently
303 ;; contains the list of positions of the enclosing open-parens.
304 ;; I.e. those positions are outside of any string/comment and the first of
305 ;; those is outside of any paren (i.e. corresponds to a nil ppss).
306 ;; If this list is empty but we are in a string or comment, then the 8th
307 ;; field contains a similar "toplevel" position.
308 (or (car (nth 9 ppss
))
311 (defsubst syntax-ppss-context
(ppss)
313 ((nth 3 ppss
) 'string
)
314 ((nth 4 ppss
) 'comment
)
317 (defvar syntax-ppss-max-span
20000
318 "Threshold below which cache info is deemed unnecessary.
319 We try to make sure that cache entries are at least this far apart
320 from each other, to avoid keeping too much useless info.")
322 (defvar syntax-begin-function nil
323 "Function to move back outside of any comment/string/paren.
324 This function should move the cursor back to some syntactically safe
325 point (where the PPSS is equivalent to nil).")
327 (defvar syntax-ppss-cache nil
328 "List of (POS . PPSS) pairs, in decreasing POS order.")
329 (make-variable-buffer-local 'syntax-ppss-cache
)
330 (defvar syntax-ppss-last nil
331 "Cache of (LAST-POS . LAST-PPSS).")
332 (make-variable-buffer-local 'syntax-ppss-last
)
334 (defalias 'syntax-ppss-after-change-function
'syntax-ppss-flush-cache
)
335 (defun syntax-ppss-flush-cache (beg &rest ignored
)
336 "Flush the cache of `syntax-ppss' starting at position BEG."
337 ;; Set syntax-propertize to refontify anything past beg.
338 (setq syntax-propertize--done
(min beg syntax-propertize--done
))
339 ;; Flush invalid cache entries.
340 (while (and syntax-ppss-cache
(> (caar syntax-ppss-cache
) beg
))
341 (setq syntax-ppss-cache
(cdr syntax-ppss-cache
)))
342 ;; Throw away `last' value if made invalid.
343 (when (< beg
(or (car syntax-ppss-last
) 0))
344 ;; If syntax-begin-function jumped to BEG, then the old state at BEG can
345 ;; depend on the text after BEG (which is presumably changed). So if
346 ;; BEG=(car (nth 10 syntax-ppss-last)) don't reuse that data because the
347 ;; assumed nil state at BEG may not be valid any more.
348 (if (<= beg
(or (syntax-ppss-toplevel-pos (cdr syntax-ppss-last
))
349 (nth 3 syntax-ppss-last
)
351 (setq syntax-ppss-last nil
)
352 (setcar syntax-ppss-last nil
)))
353 ;; Unregister if there's no cache left. Sadly this doesn't work
354 ;; because `before-change-functions' is temporarily bound to nil here.
355 ;; (unless syntax-ppss-cache
356 ;; (remove-hook 'before-change-functions 'syntax-ppss-flush-cache t))
359 (defvar syntax-ppss-stats
360 [(0 .
0.0) (0 .
0.0) (0 .
0.0) (0 .
0.0) (0 .
0.0) (1 .
2500.0)])
361 (defun syntax-ppss-stats ()
364 (cons (car x
) (truncate (/ (cdr x
) (car x
))))
368 (defun syntax-ppss (&optional pos
)
369 "Parse-Partial-Sexp State at POS, defaulting to point.
370 The returned value is the same as `parse-partial-sexp' except that
371 the 2nd and 6th values of the returned state cannot be relied upon.
372 Point is at POS when this function returns."
374 (unless pos
(setq pos
(point)))
375 (syntax-propertize pos
)
377 (let ((old-ppss (cdr syntax-ppss-last
))
378 (old-pos (car syntax-ppss-last
))
380 (pt-min (point-min)))
381 (if (and old-pos
(> old-pos pos
)) (setq old-pos nil
))
382 ;; Use the OLD-POS if usable and close. Don't update the `last' cache.
384 (if (and old-pos
(< (- pos old-pos
)
385 ;; The time to use syntax-begin-function and
386 ;; find PPSS is assumed to be about 2 * distance.
387 (* 2 (/ (cdr (aref syntax-ppss-stats
5))
388 (1+ (car (aref syntax-ppss-stats
5)))))))
390 (incf (car (aref syntax-ppss-stats
0)))
391 (incf (cdr (aref syntax-ppss-stats
0)) (- pos old-pos
))
392 (parse-partial-sexp old-pos pos nil nil old-ppss
))
395 ;; Use OLD-PPSS if possible and close enough.
396 ((and (not old-pos
) old-ppss
397 ;; If `pt-min' is too far from `pos', we could try to use
398 ;; other positions in (nth 9 old-ppss), but that doesn't
399 ;; seem to happen in practice and it would complicate this
400 ;; code (and the before-change-function code even more).
401 ;; But maybe it would be useful in "degenerate" cases such
402 ;; as when the whole file is wrapped in a set
404 (setq pt-min
(or (syntax-ppss-toplevel-pos old-ppss
)
406 (<= pt-min pos
) (< (- pos pt-min
) syntax-ppss-max-span
))
407 (incf (car (aref syntax-ppss-stats
1)))
408 (incf (cdr (aref syntax-ppss-stats
1)) (- pos pt-min
))
409 (setq ppss
(parse-partial-sexp pt-min pos
)))
410 ;; The OLD-* data can't be used. Consult the cache.
412 (let ((cache-pred nil
)
413 (cache syntax-ppss-cache
)
415 ;; I differentiate between PT-MIN and PT-BEST because
416 ;; I feel like it might be important to ensure that the
417 ;; cache is only filled with 100% sure data (whereas
418 ;; syntax-begin-function might return incorrect data).
419 ;; Maybe that's just stupid.
420 (pt-best (point-min))
422 ;; look for a usable cache entry.
423 (while (and cache
(< pos
(caar cache
)))
424 (setq cache-pred cache
)
425 (setq cache
(cdr cache
)))
426 (if cache
(setq pt-min
(caar cache
) ppss
(cdar cache
)))
428 ;; Setup the before-change function if necessary.
429 (unless (or syntax-ppss-cache syntax-ppss-last
)
430 (add-hook 'before-change-functions
431 'syntax-ppss-flush-cache t t
))
433 ;; Use the best of OLD-POS and CACHE.
434 (if (or (not old-pos
) (< old-pos pt-min
))
435 (setq pt-best pt-min ppss-best ppss
)
436 (incf (car (aref syntax-ppss-stats
4)))
437 (incf (cdr (aref syntax-ppss-stats
4)) (- pos old-pos
))
438 (setq pt-best old-pos ppss-best old-ppss
))
440 ;; Use the `syntax-begin-function' if available.
441 ;; We could try using that function earlier, but:
442 ;; - The result might not be 100% reliable, so it's better to use
443 ;; the cache if available.
444 ;; - The function might be slow.
445 ;; - If this function almost always finds a safe nearby spot,
446 ;; the cache won't be populated, so consulting it is cheap.
447 (when (and (not syntax-begin-function
)
448 (boundp 'font-lock-beginning-of-syntax-function
)
449 font-lock-beginning-of-syntax-function
)
450 (set (make-local-variable 'syntax-begin-function
)
451 font-lock-beginning-of-syntax-function
))
452 (when (and syntax-begin-function
453 (progn (goto-char pos
)
454 (funcall syntax-begin-function
)
455 ;; Make sure it's better.
457 ;; Simple sanity checks.
458 (< (point) pos
) ; backward-paragraph can fail here.
459 (not (memq (get-text-property (point) 'face
)
460 '(font-lock-string-face font-lock-doc-face
461 font-lock-comment-face
))))
462 (incf (car (aref syntax-ppss-stats
5)))
463 (incf (cdr (aref syntax-ppss-stats
5)) (- pos
(point)))
464 (setq pt-best
(point) ppss-best nil
))
467 ;; Quick case when we found a nearby pos.
468 ((< (- pos pt-best
) syntax-ppss-max-span
)
469 (incf (car (aref syntax-ppss-stats
2)))
470 (incf (cdr (aref syntax-ppss-stats
2)) (- pos pt-best
))
471 (setq ppss
(parse-partial-sexp pt-best pos nil nil ppss-best
)))
472 ;; Slow case: compute the state from some known position and
473 ;; populate the cache so we won't need to do it again soon.
475 (incf (car (aref syntax-ppss-stats
3)))
476 (incf (cdr (aref syntax-ppss-stats
3)) (- pos pt-min
))
478 ;; If `pt-min' is too far, add a few intermediate entries.
479 (while (> (- pos pt-min
) (* 2 syntax-ppss-max-span
))
480 (setq ppss
(parse-partial-sexp
481 pt-min
(setq pt-min
(/ (+ pt-min pos
) 2))
483 (let ((pair (cons pt-min ppss
)))
485 (push pair
(cdr cache-pred
))
486 (push pair syntax-ppss-cache
))))
488 ;; Compute the actual return value.
489 (setq ppss
(parse-partial-sexp pt-min pos nil nil ppss
))
492 ;; (let ((real-ppss (parse-partial-sexp (point-min) pos)))
493 ;; (setcar (last ppss 4) 0)
494 ;; (setcar (last real-ppss 4) 0)
495 ;; (setcar (last ppss 8) nil)
496 ;; (setcar (last real-ppss 8) nil)
497 ;; (unless (equal ppss real-ppss)
498 ;; (message "!!Syntax: %s != %s" ppss real-ppss)
499 ;; (setq ppss real-ppss)))
501 ;; Store it in the cache.
502 (let ((pair (cons pos ppss
)))
504 (if (> (- (caar cache-pred
) pos
) syntax-ppss-max-span
)
505 (push pair
(cdr cache-pred
))
506 (setcar cache-pred pair
))
507 (if (or (null syntax-ppss-cache
)
508 (> (- (caar syntax-ppss-cache
) pos
)
509 syntax-ppss-max-span
))
510 (push pair syntax-ppss-cache
)
511 (setcar syntax-ppss-cache pair
)))))))))
513 (setq syntax-ppss-last
(cons pos ppss
))
516 ;; If the buffer is more narrowed than when we built the cache,
517 ;; we may end up calling parse-partial-sexp with a position before
518 ;; point-min. In that case, just parse from point-min assuming
520 (parse-partial-sexp (point-min) pos
)))))
522 ;; Debugging functions
524 (defun syntax-ppss-debug ()
527 (dolist (x (append syntax-ppss-cache
(list (cons (point-min) nil
))))
528 (when pt
(push (- pt
(car x
)) min-diffs
))
532 ;; XEmacs compatibility functions
534 ;; (defun buffer-syntactic-context (&optional buffer)
535 ;; "Syntactic context at point in BUFFER.
536 ;; Either of `string', `comment' or `nil'.
537 ;; This is an XEmacs compatibility function."
538 ;; (with-current-buffer (or buffer (current-buffer))
539 ;; (syntax-ppss-context (syntax-ppss))))
541 ;; (defun buffer-syntactic-context-depth (&optional buffer)
542 ;; "Syntactic parenthesis depth at point in BUFFER.
543 ;; This is an XEmacs compatibility function."
544 ;; (with-current-buffer (or buffer (current-buffer))
545 ;; (syntax-ppss-depth (syntax-ppss))))
549 ;; arch-tag: 302f1eeb-e77c-4680-a8c5-c543e01161a5
550 ;;; syntax.el ends here