1 ;;; lex.el --- Lexical Analyzer builder
3 ;;; Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
4 ;;; 2007, 2008, 2009 Free Software Foundation, Inc.
6 ;; Author: Eric M. Ludlam <zappo@gnu.org>
8 ;; This file is part of GNU Emacs.
10 ;; GNU Emacs is free software: you can redistribute it and/or modify
11 ;; it under the terms of the GNU General Public License as published by
12 ;; the Free Software Foundation, either version 3 of the License, or
13 ;; (at your option) any later version.
15 ;; GNU Emacs is distributed in the hope that it will be useful,
16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ;; GNU General Public License for more details.
20 ;; You should have received a copy of the GNU General Public License
21 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
25 ;; This file handles the creation of lexical analyzers for different
26 ;; languages in Emacs Lisp. The purpose of a lexical analyzer is to
27 ;; convert a buffer into a list of lexical tokens. Each token
28 ;; contains the token class (such as 'number, 'symbol, 'IF, etc) and
29 ;; the location in the buffer it was found. Optionally, a token also
30 ;; contains a string representing what is at the designated buffer
33 ;; Tokens are pushed onto a token stream, which is basically a list of
34 ;; all the lexical tokens from the analyzed region. The token stream
35 ;; is then handed to the grammar which parsers the file.
39 ;; Each analyzer specifies a condition and forms. These conditions
40 ;; and forms are assembled into a function by `define-lex' that does
41 ;; the lexical analysis.
43 ;; In the lexical analyzer created with `define-lex', each condition
44 ;; is tested for a given point. When the conditin is true, the forms
47 ;; The forms can push a lexical token onto the token stream. The
48 ;; analyzer forms also must move the current analyzer point. If the
49 ;; analyzer point is moved without pushing a token, then tne matched
50 ;; syntax is effectively ignored, or skipped.
52 ;; Thus, starting at the beginning of a region to be analyzed, each
53 ;; condition is tested. One will match, and a lexical token might be
54 ;; pushed, and the point is moved to the end of the lexical token
55 ;; identified. At the new position, the process occurs again until
56 ;; the end of the specified region is reached.
58 ;;; How to use semantic-lex
60 ;; To create a lexer for a language, use the `define-lex' macro.
62 ;; The `define-lex' macro accepts a list of lexical analyzers. Each
63 ;; analyzer is created with `define-lex-analyzer', or one of the
64 ;; derivitive macros. A single analyzer defines a regular expression
65 ;; to match text in a buffer, and a short segment of code to create
68 ;; Each analyzer has a NAME, DOC, a CONDITION, and possibly some
69 ;; FORMS. The NAME is the name used in `define-lex'. The DOC
70 ;; describes what the analyzer should do.
72 ;; The CONDITION evaluates the text at the current point in the
73 ;; current buffer. If CONDITION is true, then the FORMS will be
76 ;; The purpose of the FORMS is to push new lexical tokens onto the
77 ;; list of tokens for the current buffer, and to move point after the
80 ;; Some macros for creating one analyzer are:
82 ;; define-lex-analyzer - A generic analyzer associating any style of
83 ;; condition to forms.
84 ;; define-lex-regex-analyzer - Matches a regular expression.
85 ;; define-lex-simple-regex-analyzer - Matches a regular expressions,
86 ;; and pushes the match.
87 ;; define-lex-block-analyzer - Matches list syntax, and defines
88 ;; handles open/close delimiters.
90 ;; These macros are used by the grammar compiler when lexical
91 ;; information is specified in a grammar:
92 ;; define-lex- * -type-analyzer - Matches syntax specified in
93 ;; a grammar, and pushes one token for it. The * would
94 ;; be `sexp' for things like lists or strings, and
95 ;; `string' for things that need to match some special
96 ;; string, such as "\\." where a literal match is needed.
100 ;; There are tables of different symbols managed in semantic-lex.el.
103 ;; Lexical keyword table - A Table of symbols declared in a grammar
104 ;; file with the %keyword declaration.
105 ;; Keywords are used by `semantic-lex-symbol-or-keyword'
106 ;; to create lexical tokens based on the keyword.
108 ;; Lexical type table - A table of symbols declared in a grammer
109 ;; file with the %type declaration.
110 ;; The grammar compiler uses the type table to create new
111 ;; lexical analyzers. These analyzers are then used to when
112 ;; a new lexical analyzer is made for a language.
116 ;; A lexical type defines a kind of lexical analyzer that will be
117 ;; automatically generated from a grammar file based on some
118 ;; predetermined attributes. For now these two attributes are
121 ;; * matchdatatype : define the kind of lexical analyzer. That is :
123 ;; - regexp : define a regexp analyzer (see
124 ;; `define-lex-regex-type-analyzer')
126 ;; - string : define a string analyzer (see
127 ;; `define-lex-string-type-analyzer')
129 ;; - block : define a block type analyzer (see
130 ;; `define-lex-block-type-analyzer')
132 ;; - sexp : define a sexp analyzer (see
133 ;; `define-lex-sexp-type-analyzer')
135 ;; - keyword : define a keyword analyzer (see
136 ;; `define-lex-keyword-type-analyzer')
138 ;; * syntax : define the syntax that matches a syntactic
139 ;; expression. When syntax is matched the corresponding type
140 ;; analyzer is entered and the resulting match data will be
141 ;; interpreted based on the kind of analyzer (see matchdatatype
144 ;; The following lexical types are predefined :
146 ;; +-------------+---------------+--------------------------------+
147 ;; | type | matchdatatype | syntax |
148 ;; +-------------+---------------+--------------------------------+
149 ;; | punctuation | string | "\\(\\s.\\|\\s$\\|\\s'\\)+" |
150 ;; | keyword | keyword | "\\(\\sw\\|\\s_\\)+" |
151 ;; | symbol | regexp | "\\(\\sw\\|\\s_\\)+" |
152 ;; | string | sexp | "\\s\"" |
153 ;; | number | regexp | semantic-lex-number-expression |
154 ;; | block | block | "\\s(\\|\\s)" |
155 ;; +-------------+---------------+--------------------------------+
157 ;; In a grammar you must use a %type expression to automatically generate
158 ;; the corresponding analyzers of that type.
160 ;; Here is an example to auto-generate punctuation analyzers
161 ;; with 'matchdatatype and 'syntax predefined (see table above)
163 ;; %type <punctuation> ;; will auto-generate this kind of analyzers
165 ;; It is equivalent to write :
167 ;; %type <punctuation> syntax "\\(\\s.\\|\\s$\\|\\s'\\)+" matchdatatype string
169 ;; ;; Some punctuations based on the type defines above
171 ;; %token <punctuation> NOT "!"
172 ;; %token <punctuation> NOTEQ "!="
173 ;; %token <punctuation> MOD "%"
174 ;; %token <punctuation> MODEQ "%="
177 ;;; On the Semantic 1.x lexer
179 ;; In semantic 1.x, the lexical analyzer was an all purpose routine.
180 ;; To boost efficiency, the analyzer is now a series of routines that
181 ;; are constructed at build time into a single routine. This will
182 ;; eliminate unneeded if statements to speed the lexer.
184 (require 'semantic
/fw
)
190 (if (not (fboundp 'with-syntax-table
))
192 ;; Copied from Emacs 21 for compatibility with released Emacses.
193 (defmacro with-syntax-table
(table &rest body
)
194 "With syntax table of current buffer set to a copy of TABLE, evaluate BODY.
195 The syntax table of the current buffer is saved, BODY is evaluated, and the
196 saved table is restored, even in case of an abnormal exit.
197 Value is what BODY returns."
198 (let ((old-table (make-symbol "table"))
199 (old-buffer (make-symbol "buffer")))
200 `(let ((,old-table
(syntax-table))
201 (,old-buffer
(current-buffer)))
204 (set-syntax-table (copy-syntax-table ,table
))
207 (set-buffer ,old-buffer
)
208 (set-syntax-table ,old-table
))))))
212 ;;; Semantic 2.x lexical analysis
214 (defun semantic-lex-map-symbols (fun table
&optional property
)
215 "Call function FUN on every symbol in TABLE.
216 If optional PROPERTY is non-nil, call FUN only on every symbol which
217 as a PROPERTY value. FUN receives a symbol as argument."
221 (if (or (null property
) (get symbol property
))
222 (funcall fun symbol
)))
225 ;;; Lexical keyword table handling.
227 ;; These keywords are keywords defined for using in a grammar with the
228 ;; %keyword declaration, and are not keywords used in Emacs Lisp.
230 (defvar semantic-flex-keywords-obarray nil
231 "Buffer local keyword obarray for the lexical analyzer.
232 These keywords are matched explicitly, and converted into special symbols.")
233 (make-variable-buffer-local 'semantic-flex-keywords-obarray
)
235 (defmacro semantic-lex-keyword-invalid
(name)
236 "Signal that NAME is an invalid keyword name."
237 `(signal 'wrong-type-argument
'(semantic-lex-keyword-p ,name
)))
239 (defsubst semantic-lex-keyword-symbol
(name)
240 "Return keyword symbol with NAME or nil if not found."
241 (and (arrayp semantic-flex-keywords-obarray
)
243 (intern-soft name semantic-flex-keywords-obarray
)))
245 (defsubst semantic-lex-keyword-p
(name)
246 "Return non-nil if a keyword with NAME exists in the keyword table.
247 Return nil otherwise."
248 (and (setq name
(semantic-lex-keyword-symbol name
))
249 (symbol-value name
)))
251 (defsubst semantic-lex-keyword-set
(name value
)
252 "Set value of keyword with NAME to VALUE and return VALUE."
253 (set (intern name semantic-flex-keywords-obarray
) value
))
255 (defsubst semantic-lex-keyword-value
(name)
256 "Return value of keyword with NAME.
257 Signal an error if a keyword with NAME does not exist."
258 (let ((keyword (semantic-lex-keyword-symbol name
)))
260 (symbol-value keyword
)
261 (semantic-lex-keyword-invalid name
))))
263 (defsubst semantic-lex-keyword-put
(name property value
)
264 "For keyword with NAME, set its PROPERTY to VALUE."
265 (let ((keyword (semantic-lex-keyword-symbol name
)))
267 (put keyword property value
)
268 (semantic-lex-keyword-invalid name
))))
270 (defsubst semantic-lex-keyword-get
(name property
)
271 "For keyword with NAME, return its PROPERTY value."
272 (let ((keyword (semantic-lex-keyword-symbol name
)))
274 (get keyword property
)
275 (semantic-lex-keyword-invalid name
))))
277 (defun semantic-lex-make-keyword-table (specs &optional propspecs
)
278 "Convert keyword SPECS into an obarray and return it.
279 SPECS must be a list of (NAME . TOKSYM) elements, where:
281 NAME is the name of the keyword symbol to define.
282 TOKSYM is the lexical token symbol of that keyword.
284 If optional argument PROPSPECS is non nil, then interpret it, and
285 apply those properties.
286 PROPSPECS must be a list of (NAME PROPERTY VALUE) elements."
287 ;; Create the symbol hash table
288 (let ((semantic-flex-keywords-obarray (make-vector 13 0))
290 ;; fill it with stuff
292 (setq spec
(car specs
)
294 (semantic-lex-keyword-set (car spec
) (cdr spec
)))
295 ;; Apply all properties
297 (setq spec
(car propspecs
)
298 propspecs
(cdr propspecs
))
299 (semantic-lex-keyword-put (car spec
) (nth 1 spec
) (nth 2 spec
)))
300 semantic-flex-keywords-obarray
))
302 (defsubst semantic-lex-map-keywords
(fun &optional property
)
303 "Call function FUN on every lexical keyword.
304 If optional PROPERTY is non-nil, call FUN only on every keyword which
305 as a PROPERTY value. FUN receives a lexical keyword as argument."
306 (semantic-lex-map-symbols
307 fun semantic-flex-keywords-obarray property
))
309 (defun semantic-lex-keywords (&optional property
)
310 "Return a list of lexical keywords.
311 If optional PROPERTY is non-nil, return only keywords which have a
314 (semantic-lex-map-keywords
315 #'(lambda (symbol) (setq keywords
(cons symbol keywords
)))
319 ;;; Inline functions:
321 (defvar semantic-lex-unterminated-syntax-end-function
)
322 (defvar semantic-lex-analysis-bounds
)
323 (defvar semantic-lex-end-point
)
325 (defsubst semantic-lex-token-bounds
(token)
326 "Fetch the start and end locations of the lexical token TOKEN.
327 Return a pair (START . END)."
328 (if (not (numberp (car (cdr token
))))
332 (defsubst semantic-lex-token-start
(token)
333 "Fetch the start position of the lexical token TOKEN.
334 See also the function `semantic-lex-token'."
335 (car (semantic-lex-token-bounds token
)))
337 (defsubst semantic-lex-token-end
(token)
338 "Fetch the end position of the lexical token TOKEN.
339 See also the function `semantic-lex-token'."
340 (cdr (semantic-lex-token-bounds token
)))
342 (defsubst semantic-lex-unterminated-syntax-detected
(syntax)
343 "Inside a lexical analyzer, use this when unterminated syntax was found.
344 Argument SYNTAX indicates the type of syntax that is unterminated.
345 The job of this function is to move (point) to a new logical location
346 so that analysis can continue, if possible."
348 (funcall semantic-lex-unterminated-syntax-end-function
350 (car semantic-lex-analysis-bounds
)
351 (cdr semantic-lex-analysis-bounds
)
353 (setq semantic-lex-end-point
(point)))
355 ;;; Type table handling.
357 ;; The lexical type table manages types that occur in a grammar file
358 ;; with the %type declaration. Types represent different syntaxes.
359 ;; See code for `semantic-lex-preset-default-types' for the classic
361 (defvar semantic-lex-types-obarray nil
362 "Buffer local types obarray for the lexical analyzer.")
363 (make-variable-buffer-local 'semantic-lex-types-obarray
)
365 (defmacro semantic-lex-type-invalid
(type)
366 "Signal that TYPE is an invalid lexical type name."
367 `(signal 'wrong-type-argument
'(semantic-lex-type-p ,type
)))
369 (defsubst semantic-lex-type-symbol
(type)
370 "Return symbol with TYPE or nil if not found."
371 (and (arrayp semantic-lex-types-obarray
)
373 (intern-soft type semantic-lex-types-obarray
)))
375 (defsubst semantic-lex-type-p
(type)
376 "Return non-nil if a symbol with TYPE name exists."
377 (and (setq type
(semantic-lex-type-symbol type
))
378 (symbol-value type
)))
380 (defsubst semantic-lex-type-set
(type value
)
381 "Set value of symbol with TYPE name to VALUE and return VALUE."
382 (set (intern type semantic-lex-types-obarray
) value
))
384 (defsubst semantic-lex-type-value
(type &optional noerror
)
385 "Return value of symbol with TYPE name.
386 If optional argument NOERROR is non-nil return nil if a symbol with
387 TYPE name does not exist. Otherwise signal an error."
388 (let ((sym (semantic-lex-type-symbol type
)))
392 (semantic-lex-type-invalid type
)))))
394 (defsubst semantic-lex-type-put
(type property value
&optional add
)
395 "For symbol with TYPE name, set its PROPERTY to VALUE.
396 If optional argument ADD is non-nil, create a new symbol with TYPE
397 name if it does not already exist. Otherwise signal an error."
398 (let ((sym (semantic-lex-type-symbol type
)))
400 (or add
(semantic-lex-type-invalid type
))
401 (semantic-lex-type-set type nil
)
402 (setq sym
(semantic-lex-type-symbol type
)))
403 (put sym property value
)))
405 (defsubst semantic-lex-type-get
(type property
&optional noerror
)
406 "For symbol with TYPE name, return its PROPERTY value.
407 If optional argument NOERROR is non-nil return nil if a symbol with
408 TYPE name does not exist. Otherwise signal an error."
409 (let ((sym (semantic-lex-type-symbol type
)))
413 (semantic-lex-type-invalid type
)))))
415 (defun semantic-lex-preset-default-types ()
416 "Install useful default properties for well known types."
417 (semantic-lex-type-put "punctuation" 'matchdatatype
'string t
)
418 (semantic-lex-type-put "punctuation" 'syntax
"\\(\\s.\\|\\s$\\|\\s'\\)+")
419 (semantic-lex-type-put "keyword" 'matchdatatype
'keyword t
)
420 (semantic-lex-type-put "keyword" 'syntax
"\\(\\sw\\|\\s_\\)+")
421 (semantic-lex-type-put "symbol" 'matchdatatype
'regexp t
)
422 (semantic-lex-type-put "symbol" 'syntax
"\\(\\sw\\|\\s_\\)+")
423 (semantic-lex-type-put "string" 'matchdatatype
'sexp t
)
424 (semantic-lex-type-put "string" 'syntax
"\\s\"")
425 (semantic-lex-type-put "number" 'matchdatatype
'regexp t
)
426 (semantic-lex-type-put "number" 'syntax
'semantic-lex-number-expression
)
427 (semantic-lex-type-put "block" 'matchdatatype
'block t
)
428 (semantic-lex-type-put "block" 'syntax
"\\s(\\|\\s)")
431 (defun semantic-lex-make-type-table (specs &optional propspecs
)
432 "Convert type SPECS into an obarray and return it.
433 SPECS must be a list of (TYPE . TOKENS) elements, where:
435 TYPE is the name of the type symbol to define.
436 TOKENS is an list of (TOKSYM . MATCHER) elements, where:
438 TOKSYM is any lexical token symbol.
439 MATCHER is a string or regexp a text must match to be a such
442 If optional argument PROPSPECS is non nil, then interpret it, and
443 apply those properties.
444 PROPSPECS must be a list of (TYPE PROPERTY VALUE)."
445 ;; Create the symbol hash table
446 (let* ((semantic-lex-types-obarray (make-vector 13 0))
447 spec type tokens token alist default
)
448 ;; fill it with stuff
450 (setq spec
(car specs
)
457 (setq token
(car tokens
)
460 (setq alist
(cons token alist
))
461 (setq token
(car token
))
464 "*Warning* default value of <%s> tokens changed to %S, was %S"
466 (setq default token
)))
467 ;; Ensure the default matching spec is the first one.
468 (semantic-lex-type-set type
(cons default
(nreverse alist
))))
469 ;; Install useful default types & properties
470 (semantic-lex-preset-default-types)
471 ;; Apply all properties
473 (setq spec
(car propspecs
)
474 propspecs
(cdr propspecs
))
475 ;; Create the type if necessary.
476 (semantic-lex-type-put (car spec
) (nth 1 spec
) (nth 2 spec
) t
))
477 semantic-lex-types-obarray
))
479 (defsubst semantic-lex-map-types
(fun &optional property
)
480 "Call function FUN on every lexical type.
481 If optional PROPERTY is non-nil, call FUN only on every type symbol
482 which as a PROPERTY value. FUN receives a type symbol as argument."
483 (semantic-lex-map-symbols
484 fun semantic-lex-types-obarray property
))
486 (defun semantic-lex-types (&optional property
)
487 "Return a list of lexical type symbols.
488 If optional PROPERTY is non-nil, return only type symbols which have
491 (semantic-lex-map-types
492 #'(lambda (symbol) (setq types
(cons symbol types
)))
496 ;;; Lexical Analyzer framework settings
499 (defvar semantic-lex-analyzer
'semantic-flex
500 "The lexical analyzer used for a given buffer.
501 See `semantic-lex' for documentation.
502 For compatibility with Semantic 1.x it defaults to `semantic-flex'.")
503 (make-variable-buffer-local 'semantic-lex-analyzer
)
505 (defvar semantic-lex-tokens
519 "An alist of of semantic token types.
520 As of December 2001 (semantic 1.4beta13), this variable is not used in
521 any code. The only use is to refer to the doc-string from elsewhere.
523 The key to this alist is the symbol representing token type that
524 \\[semantic-flex] returns. These are
526 - bol: Empty string matching a beginning of line.
527 This token is produced with
528 `semantic-lex-beginning-of-line'.
530 - charquote: String sequences that match `\\s\\+' regexp.
531 This token is produced with `semantic-lex-charquote'.
533 - close-paren: Characters that match `\\s)' regexp.
534 These are typically `)', `}', `]', etc.
535 This token is produced with
536 `semantic-lex-close-paren'.
538 - comment: A comment chunk. These token types are not
540 This token is produced with `semantic-lex-comments'.
541 Comments are ignored with `semantic-lex-ignore-comments'.
542 Comments are treated as whitespace with
543 `semantic-lex-comments-as-whitespace'.
545 - newline Characters matching `\\s-*\\(\n\\|\\s>\\)' regexp.
546 This token is produced with `semantic-lex-newline'.
548 - open-paren: Characters that match `\\s(' regexp.
549 These are typically `(', `{', `[', etc.
550 If `semantic-lex-paren-or-list' is used,
551 then `open-paren' is not usually generated unless
552 the `depth' argument to \\[semantic-lex] is
554 This token is always produced if the analyzer
555 `semantic-lex-open-paren' is used.
557 - punctuation: Characters matching `{\\(\\s.\\|\\s$\\|\\s'\\)'
559 This token is produced with `semantic-lex-punctuation'.
560 Always specify this analyzer after the comment
563 - semantic-list: String delimited by matching parenthesis, braces,
564 etc. that the lexer skipped over, because the
565 `depth' parameter to \\[semantic-flex] was not high
567 This token is produced with `semantic-lex-paren-or-list'.
569 - string: Quoted strings, i.e., string sequences that start
570 and end with characters matching `\\s\"'
571 regexp. The lexer relies on @code{forward-sexp} to
572 find the matching end.
573 This token is produced with `semantic-lex-string'.
575 - symbol: String sequences that match `\\(\\sw\\|\\s_\\)+'
577 This token is produced with
578 `semantic-lex-symbol-or-keyword'. Always add this analyzer
579 after `semantic-lex-number', or other analyzers that
580 match its regular expression.
582 - whitespace: Characters that match `\\s-+' regexp.
583 This token is produced with `semantic-lex-whitespace'.")
585 (defvar semantic-lex-syntax-modifications nil
586 "Changes to the syntax table for this buffer.
587 These changes are active only while the buffer is being flexed.
588 This is a list where each element has the form:
590 CHAR is the char passed to `modify-syntax-entry',
591 and CLASS is the string also passed to `modify-syntax-entry' to define
592 what syntax class CHAR has.")
593 (make-variable-buffer-local 'semantic-lex-syntax-modifications
)
595 (defvar semantic-lex-syntax-table nil
596 "Syntax table used by lexical analysis.
597 See also `semantic-lex-syntax-modifications'.")
598 (make-variable-buffer-local 'semantic-lex-syntax-table
)
600 (defvar semantic-lex-comment-regex nil
601 "Regular expression for identifying comment start during lexical analysis.
602 This may be automatically set when semantic initializes in a mode, but
603 may need to be overriden for some special languages.")
604 (make-variable-buffer-local 'semantic-lex-comment-regex
)
606 (defvar semantic-lex-number-expression
607 ;; This expression was written by David Ponce for Java, and copied
608 ;; here for C and any other similar language.
611 "\\<[0-9]+[.][0-9]+\\([eE][-+]?[0-9]+\\)?[fFdD]?\\>"
613 "\\<[0-9]+[.][eE][-+]?[0-9]+[fFdD]?\\>"
615 "\\<[0-9]+[.][fFdD]\\>"
619 "[.][0-9]+\\([eE][-+]?[0-9]+\\)?[fFdD]?\\>"
621 "\\<[0-9]+[eE][-+]?[0-9]+[fFdD]?\\>"
623 "\\<0[xX][0-9a-fA-F]+[lL]?\\>"
625 "\\<[0-9]+[lLfFdD]?\\>"
628 "Regular expression for matching a number.
629 If this value is nil, no number extraction is done during lex.
630 This expression tries to match C and Java like numbers.
642 <DECIMAL_LITERAL>[lL]?
644 | <OCTAL_LITERAL>[lL]?
649 FLOATING_POINT_LITERAL:
650 [0-9]+[.][0-9]*<EXPONENT>?[fFdD]?
651 | [.][0-9]+<EXPONENT>?[fFdD]?
652 | [0-9]+<EXPONENT>[fFdD]?
653 | [0-9]+<EXPONENT>?[fFdD]
655 (make-variable-buffer-local 'semantic-lex-number-expression
)
657 (defvar semantic-lex-depth
0
658 "Default lexing depth.
659 This specifies how many lists to create tokens in.")
660 (make-variable-buffer-local 'semantic-lex-depth
)
662 (defvar semantic-lex-unterminated-syntax-end-function
663 (lambda (syntax syntax-start lex-end
) lex-end
)
664 "Function called when unterminated syntax is encountered.
665 This should be set to one function. That function should take three
666 parameters. The SYNTAX, or type of syntax which is unterminated.
667 SYNTAX-START where the broken syntax begins.
668 LEX-END is where the lexical analysis was asked to end.
669 This function can be used for languages that can intelligently fix up
670 broken syntax, or the exit lexical analysis via `throw' or `signal'
671 when finding unterminated syntax.")
673 ;;; Interactive testing commands
675 (declare-function semantic-elapsed-time
"semantic")
677 (defun semantic-lex-test (arg)
678 "Test the semantic lexer in the current buffer.
679 If universal argument ARG, then try the whole buffer."
682 (let* ((start (current-time))
683 (result (semantic-lex
684 (if arg
(point-min) (point))
686 (end (current-time)))
687 (message "Elapsed Time: %.2f seconds."
688 (semantic-elapsed-time start end
))
689 (pop-to-buffer "*Lexer Output*")
692 (insert (pp-to-string result
))
693 (goto-char (point-min))
696 (defvar semantic-lex-debug nil
697 "When non-nil, debug the local lexical analyzer.")
699 (defun semantic-lex-debug (arg)
700 "Debug the semantic lexer in the current buffer.
701 Argument ARG specifies of the analyze the whole buffer, or start at point.
702 While engaged, each token identified by the lexer will be highlighted
703 in the target buffer A description of the current token will be
704 displayed in the minibuffer. Press SPC to move to the next lexical token."
706 (require 'semantic
/debug
)
707 (let ((semantic-lex-debug t
))
708 (semantic-lex-test arg
)))
710 (defun semantic-lex-highlight-token (token)
711 "Highlight the lexical TOKEN.
712 TOKEN is a lexical token with a START And END position.
714 (let ((o (semantic-make-overlay (semantic-lex-token-start token
)
715 (semantic-lex-token-end token
))))
716 (semantic-overlay-put o
'face
'highlight
)
719 (defsubst semantic-lex-debug-break
(token)
720 "Break during lexical analysis at TOKEN."
721 (when semantic-lex-debug
726 (setq o
(semantic-lex-highlight-token token
)))
728 (format "%S :: SPC - continue" token
))
731 (semantic-overlay-delete o
))))))
733 ;;; Lexical analyzer creation
735 ;; Code for creating a lex function from lists of analyzers.
737 ;; A lexical analyzer is created from a list of individual analyzers.
738 ;; Each individual analyzer specifies a single match, and code that
741 ;; Creation of an analyzer assembles these analyzers into a new function
742 ;; with the behaviors of all the individual analyzers.
744 (defmacro semantic-lex-one-token
(analyzers)
745 "Calculate one token from the current buffer at point.
746 Uses locally bound variables from `define-lex'.
747 Argument ANALYZERS is the list of analyzers being used."
748 (cons 'cond
(mapcar #'symbol-value analyzers
)))
750 (defvar semantic-lex-end-point nil
751 "The end point as tracked through lexical functions.")
753 (defvar semantic-lex-current-depth nil
754 "The current depth as tracked through lexical functions.")
756 (defvar semantic-lex-maximum-depth nil
757 "The maximum depth of parenthisis as tracked through lexical functions.")
759 (defvar semantic-lex-token-stream nil
760 "The current token stream we are collecting.")
762 (defvar semantic-lex-analysis-bounds nil
763 "The bounds of the current analysis.")
765 (defvar semantic-lex-block-streams nil
766 "Streams of tokens inside collapsed blocks.
767 This is an alist of (ANCHOR . STREAM) elements where ANCHOR is the
768 start position of the block, and STREAM is the list of tokens in that
771 (defvar semantic-lex-reset-hooks nil
772 "Abnormal hook used by major-modes to reset lexical analyzers.
773 Hook functions are called with START and END values for the
774 current lexical pass. Should be set with `add-hook', specifying
777 ;; Stack of nested blocks.
778 (defvar semantic-lex-block-stack nil
)
779 ;;(defvar semantic-lex-timeout 5
780 ;; "*Number of sections of lexing before giving up.")
782 (defmacro define-lex
(name doc
&rest analyzers
)
783 "Create a new lexical analyzer with NAME.
784 DOC is a documentation string describing this analyzer.
785 ANALYZERS are small code snippets of analyzers to use when
786 building the new NAMED analyzer. Only use analyzers which
787 are written to be used in `define-lex'.
788 Each analyzer should be an analyzer created with `define-lex-analyzer'.
789 Note: The order in which analyzers are listed is important.
790 If two analyzers can match the same text, it is important to order the
791 analyzers so that the one you want to match first occurs first. For
792 example, it is good to put a numbe analyzer in front of a symbol
793 analyzer which might mistake a number for as a symbol."
794 `(defun ,name
(start end
&optional depth length
)
795 ,(concat doc
"\nSee `semantic-lex' for more information.")
796 ;; Make sure the state of block parsing starts over.
797 (setq semantic-lex-block-streams nil
)
798 ;; Allow specialty reset items.
799 (run-hook-with-args 'semantic-lex-reset-hooks start end
)
801 (let* (;(starttime (current-time))
802 (starting-position (point))
803 (semantic-lex-token-stream nil
)
804 (semantic-lex-block-stack nil
)
806 (semantic-lex-end-point start
)
807 (semantic-lex-current-depth 0)
808 ;; Use the default depth when not specified.
809 (semantic-lex-maximum-depth
810 (or depth semantic-lex-depth
))
811 ;; Bounds needed for unterminated syntax
812 (semantic-lex-analysis-bounds (cons start end
))
813 ;; This entry prevents text properties from
814 ;; confusing our lexical analysis. See Emacs 22 (CVS)
815 ;; version of C++ mode with template hack text properties.
816 (parse-sexp-lookup-properties nil
)
818 ;; Maybe REMOVE THIS LATER.
819 ;; Trying to find incremental parser bug.
820 (when (> end
(point-max))
821 (error ,(format "%s: end (%%d) > point-max (%%d)" name
)
823 (with-syntax-table semantic-lex-syntax-table
825 (while (and (< (point) end
)
827 (<= (length semantic-lex-token-stream
) length
)))
828 (semantic-lex-one-token ,analyzers
)
829 (when (eq semantic-lex-end-point tmp-start
)
830 (error ,(format "%s: endless loop at %%d, after %%S" name
)
831 tmp-start
(car semantic-lex-token-stream
)))
832 (setq tmp-start semantic-lex-end-point
)
833 (goto-char semantic-lex-end-point
)
834 ;;(when (> (semantic-elapsed-time starttime (current-time))
835 ;; semantic-lex-timeout)
836 ;; (error "Timeout during lex at char %d" (point)))
837 (semantic-throw-on-input 'lex
)
838 (semantic-lex-debug-break (car semantic-lex-token-stream
))
840 ;; Check that there is no unterminated block.
841 (when semantic-lex-block-stack
842 (let* ((last (pop semantic-lex-block-stack
))
846 ,(format "%s: `%%s' block from %%S is unterminated" name
)
847 (car blk
) (cadr blk
))
848 (setq blk
(pop semantic-lex-block-stack
)))
849 (semantic-lex-unterminated-syntax-detected (car last
))))
850 ;; Return to where we started.
851 ;; Do not wrap in protective stuff so that if there is an error
852 ;; thrown, the user knows where.
853 (goto-char starting-position
)
854 ;; Return the token stream
855 (nreverse semantic-lex-token-stream
))))
857 ;;; Collapsed block tokens delimited by any tokens.
859 (defun semantic-lex-start-block (syntax)
860 "Mark the last read token as the beginning of a SYNTAX block."
861 (if (or (not semantic-lex-maximum-depth
)
862 (< semantic-lex-current-depth semantic-lex-maximum-depth
))
863 (setq semantic-lex-current-depth
(1+ semantic-lex-current-depth
))
864 (push (list syntax
(car semantic-lex-token-stream
))
865 semantic-lex-block-stack
)))
867 (defun semantic-lex-end-block (syntax)
868 "Process the end of a previously marked SYNTAX block.
869 That is, collapse the tokens inside that block, including the
870 beginning and end of block tokens, into a high level block token of
872 The token at beginning of block is the one marked by a previous call
873 to `semantic-lex-start-block'. The current token is the end of block.
874 The collapsed tokens are saved in `semantic-lex-block-streams'."
875 (if (null semantic-lex-block-stack
)
876 (setq semantic-lex-current-depth
(1- semantic-lex-current-depth
))
877 (let* ((stream semantic-lex-token-stream
)
878 (blk (pop semantic-lex-block-stack
))
880 (first (car bstream
))
881 (last (pop stream
)) ;; The current token mark the EOBLK
883 (if (not (eq (car blk
) syntax
))
884 ;; SYNTAX doesn't match the syntax of the current block in
885 ;; the stack. So we encountered the end of the SYNTAX block
886 ;; before the end of the current one in the stack which is
887 ;; signaled unterminated.
888 (semantic-lex-unterminated-syntax-detected (car blk
))
889 ;; Move tokens found inside the block from the main stream
890 ;; into a separate block stream.
891 (while (and stream
(not (eq (setq tok
(pop stream
)) first
)))
893 ;; The token marked as beginning of block was not encountered.
894 ;; This should not happen!
896 (error "Token %S not found at beginning of block `%s'"
898 ;; Save the block stream for future reuse, to avoid to redo
899 ;; the lexical analysis of the block content!
900 ;; Anchor the block stream with its start position, so we can
901 ;; use: (cdr (assq start semantic-lex-block-streams)) to
902 ;; quickly retrieve the lexical stream associated to a block.
903 (setcar blk
(semantic-lex-token-start first
))
904 (setcdr blk
(nreverse bstream
))
905 (push blk semantic-lex-block-streams
)
906 ;; In the main stream, replace the tokens inside the block by
907 ;; a high level block token of class SYNTAX.
908 (setq semantic-lex-token-stream stream
)
909 (semantic-lex-push-token
911 syntax
(car blk
) (semantic-lex-token-end last
)))
914 ;;; Lexical token API
916 ;; Functions for accessing parts of a token. Use these functions
917 ;; instead of accessing the list structure directly because the
918 ;; contents of the lexical may change.
920 (defmacro semantic-lex-token
(symbol start end
&optional str
)
921 "Create a lexical token.
922 SYMBOL is a symbol representing the class of syntax found.
923 START and END define the bounds of the token in the current buffer.
924 Optional STR is the string for the token iff the the bounds
925 in the buffer do not cover the string they represent. (As from
927 ;; This if statement checks the existance of a STR argument at
928 ;; compile time, where STR is some symbol or constant. If the
929 ;; variable STr (runtime) is nil, this will make an incorrect decision.
931 ;; It is like this to maintain the original speed of the compiled
934 `(cons ,symbol
(cons ,str
(cons ,start
,end
)))
935 `(cons ,symbol
(cons ,start
,end
))))
937 (defun semantic-lex-token-p (thing)
938 "Return non-nil if THING is a semantic lex token.
939 This is an exhaustively robust check."
941 (symbolp (car thing
))
942 (or (and (numberp (nth 1 thing
))
943 (numberp (nthcdr 2 thing
)))
944 (and (stringp (nth 1 thing
))
945 (numberp (nth 2 thing
))
946 (numberp (nthcdr 3 thing
)))
950 (defun semantic-lex-token-with-text-p (thing)
951 "Return non-nil if THING is a semantic lex token.
952 This is an exhaustively robust check."
954 (symbolp (car thing
))
956 (stringp (nth 1 thing
))
957 (numberp (nth 2 thing
))
958 (numberp (nth 3 thing
)))
961 (defun semantic-lex-token-without-text-p (thing)
962 "Return non-nil if THING is a semantic lex token.
963 This is an exhaustively robust check."
965 (symbolp (car thing
))
967 (numberp (nth 1 thing
))
968 (numberp (nth 2 thing
)))
973 (defun semantic-lex-expand-block-specs (specs)
974 "Expand block specifications SPECS into a Lisp form.
975 SPECS is a list of (BLOCK BEGIN END) elements where BLOCK, BEGIN, and
976 END are token class symbols that indicate to produce one collapsed
977 BLOCK token from tokens found between BEGIN and END ones.
978 BLOCK must be a non-nil symbol, and at least one of the BEGIN or END
979 symbols must be non-nil too.
980 When BEGIN is non-nil, generate a call to `semantic-lex-start-block'
981 when a BEGIN token class is encountered.
982 When END is non-nil, generate a call to `semantic-lex-end-block' when
983 an END token class is encountered."
984 (let ((class (make-symbol "class"))
989 (push `((eq ',(nth 1 spec
) ,class
)
990 (semantic-lex-start-block ',(car spec
)))
993 (push `((eq ',(nth 2 spec
) ,class
)
994 (semantic-lex-end-block ',(car spec
)))
997 `((let ((,class
(semantic-lex-token-class
998 (car semantic-lex-token-stream
))))
999 (cond ,@(nreverse form
))))
1003 (defmacro semantic-lex-push-token
(token &rest blockspecs
)
1004 "Push TOKEN in the lexical analyzer token stream.
1005 Return the lexical analysis current end point.
1006 If optional arguments BLOCKSPECS is non-nil, it specifies to process
1007 collapsed block tokens. See `semantic-lex-expand-block-specs' for
1009 This macro should only be called within the bounds of
1010 `define-lex-analyzer'. It changes the values of the lexical analyzer
1011 variables `token-stream' and `semantic-lex-end-point'. If you need to
1012 move `semantic-lex-end-point' somewhere else, just modify this
1013 variable after calling `semantic-lex-push-token'."
1015 (push ,token semantic-lex-token-stream
)
1016 ,@(semantic-lex-expand-block-specs blockspecs
)
1017 (setq semantic-lex-end-point
1018 (semantic-lex-token-end (car semantic-lex-token-stream
)))
1021 (defsubst semantic-lex-token-class
(token)
1022 "Fetch the class of the lexical token TOKEN.
1023 See also the function `semantic-lex-token'."
1026 (defsubst semantic-lex-token-text
(token)
1027 "Fetch the text associated with the lexical token TOKEN.
1028 See also the function `semantic-lex-token'."
1029 (if (stringp (car (cdr token
)))
1031 (buffer-substring-no-properties
1032 (semantic-lex-token-start token
)
1033 (semantic-lex-token-end token
))))
1035 (defun semantic-lex-init ()
1036 "Initialize any lexical state for this buffer."
1037 (unless semantic-lex-comment-regex
1038 (setq semantic-lex-comment-regex
1039 (if comment-start-skip
1040 (concat "\\(\\s<\\|" comment-start-skip
"\\)")
1042 ;; Setup the lexer syntax-table
1043 (setq semantic-lex-syntax-table
(copy-syntax-table (syntax-table)))
1044 (dolist (mod semantic-lex-syntax-modifications
)
1045 (modify-syntax-entry
1046 (car mod
) (nth 1 mod
) semantic-lex-syntax-table
)))
1049 (define-overloadable-function semantic-lex
(start end
&optional depth length
)
1050 "Lexically analyze text in the current buffer between START and END.
1051 Optional argument DEPTH indicates at what level to scan over entire
1052 lists. The last argument, LENGTH specifies that `semantic-lex'
1053 should only return LENGTH tokens. The return value is a token stream.
1054 Each element is a list, such of the form
1055 (symbol start-expression . end-expression)
1056 where SYMBOL denotes the token type.
1057 See `semantic-lex-tokens' variable for details on token types. END
1058 does not mark the end of the text scanned, only the end of the
1059 beginning of text scanned. Thus, if a string extends past END, the
1060 end of the return token will be larger than END. To truly restrict
1061 scanning, use `narrow-to-region'."
1062 (funcall semantic-lex-analyzer start end depth length
))
1064 (defsubst semantic-lex-buffer
(&optional depth
)
1065 "Lex the current buffer.
1066 Optional argument DEPTH is the depth to scan into lists."
1067 (semantic-lex (point-min) (point-max) depth
))
1069 (defsubst semantic-lex-list
(semlist depth
)
1070 "Lex the body of SEMLIST to DEPTH."
1071 (semantic-lex (semantic-lex-token-start semlist
)
1072 (semantic-lex-token-end semlist
)
1075 ;;; Analyzer creation macros
1077 ;; An individual analyzer is a condition and code that goes with it.
1079 ;; Created analyzers become variables with the code associated with them
1080 ;; as the symbol value. These analyzers are assembled into a lexer
1081 ;; to create new lexical analyzers.
1083 (defcustom semantic-lex-debug-analyzers nil
1084 "Non nil means to debug analyzers with syntax protection.
1085 Only in effect if `debug-on-error' is also non-nil."
1089 (defmacro semantic-lex-unterminated-syntax-protection
(syntax &rest forms
)
1090 "For SYNTAX, execute FORMS with protection for unterminated syntax.
1091 If FORMS throws an error, treat this as a syntax problem, and
1092 execute the unterminated syntax code. FORMS should return a position.
1093 Irreguardless of an error, the cursor should be moved to the end of
1094 the desired syntax, and a position returned.
1095 If `debug-on-error' is set, errors are not caught, so that you can
1097 Avoid using a large FORMS since it is duplicated."
1098 `(if (and debug-on-error semantic-lex-debug-analyzers
)
1103 (semantic-lex-unterminated-syntax-detected ,syntax
)))))
1104 (put 'semantic-lex-unterminated-syntax-protection
1105 'lisp-indent-function
1)
1107 (defmacro define-lex-analyzer
(name doc condition
&rest forms
)
1108 "Create a single lexical analyzer NAME with DOC.
1109 When an analyzer is called, the current buffer and point are
1110 positioned in a buffer at the location to be analyzed.
1111 CONDITION is an expression which returns t if FORMS should be run.
1112 Within the bounds of CONDITION and FORMS, the use of backquote
1113 can be used to evaluate expressions at compile time.
1114 While forms are running, the following variables will be locally bound:
1115 `semantic-lex-analysis-bounds' - The bounds of the current analysis.
1116 of the form (START . END)
1117 `semantic-lex-maximum-depth' - The maximum depth of semantic-list
1118 for the current analysis.
1119 `semantic-lex-current-depth' - The current depth of `semantic-list' that has
1121 `semantic-lex-end-point' - End Point after match.
1122 Analyzers should set this to a buffer location if their
1123 match string does not represent the end of the matched text.
1124 `semantic-lex-token-stream' - The token list being collected.
1125 Add new lexical tokens to this list.
1126 Proper action in FORMS is to move the value of `semantic-lex-end-point' to
1127 after the location of the analyzed entry, and to add any discovered tokens
1128 at the beginning of `semantic-lex-token-stream'.
1129 This can be done by using `semantic-lex-push-token'."
1131 (defvar ,name nil
,doc
)
1133 ;; Do this part separately so that re-evaluation rebuilds this code.
1134 (setq ,name
'(,condition
,@forms
))
1135 ;; Build a single lexical analyzer function, so the doc for
1136 ;; function help is automatically provided, and perhaps the
1137 ;; function could be useful for testing and debugging one
1139 (fset ',name
(lambda () ,doc
1140 (let ((semantic-lex-token-stream nil
)
1141 (semantic-lex-end-point (point))
1142 (semantic-lex-analysis-bounds
1143 (cons (point) (point-max)))
1144 (semantic-lex-current-depth 0)
1145 (semantic-lex-maximum-depth
1148 (when ,condition
,@forms
)
1149 semantic-lex-token-stream
)))
1152 (defmacro define-lex-regex-analyzer
(name doc regexp
&rest forms
)
1153 "Create a lexical analyzer with NAME and DOC that will match REGEXP.
1154 FORMS are evaluated upon a successful match.
1155 See `define-lex-analyzer' for more about analyzers."
1156 `(define-lex-analyzer ,name
1158 (looking-at ,regexp
)
1162 (defmacro define-lex-simple-regex-analyzer
(name doc regexp toksym
1165 "Create a lexical analyzer with NAME and DOC that match REGEXP.
1166 TOKSYM is the symbol to use when creating a semantic lexical token.
1167 INDEX is the index into the match that defines the bounds of the token.
1168 Index should be a plain integer, and not specified in the macro as an
1170 FORMS are evaluated upon a successful match BEFORE the new token is
1171 created. It is valid to ignore FORMS.
1172 See `define-lex-analyzer' for more about analyzers."
1173 `(define-lex-analyzer ,name
1175 (looking-at ,regexp
)
1177 (semantic-lex-push-token
1178 (semantic-lex-token ,toksym
1179 (match-beginning ,(or index
0))
1180 (match-end ,(or index
0))))
1183 (defmacro define-lex-block-analyzer
(name doc spec1
&rest specs
)
1184 "Create a lexical analyzer NAME for paired delimiters blocks.
1185 It detects a paired delimiters block or the corresponding open or
1186 close delimiter depending on the value of the variable
1187 `semantic-lex-current-depth'. DOC is the documentation string of the lexical
1188 analyzer. SPEC1 and SPECS specify the token symbols and open, close
1189 delimiters used. Each SPEC has the form:
1191 \(BLOCK-SYM (OPEN-DELIM OPEN-SYM) (CLOSE-DELIM CLOSE-SYM))
1193 where BLOCK-SYM is the symbol returned in a block token. OPEN-DELIM
1194 and CLOSE-DELIM are respectively the open and close delimiters
1195 identifying a block. OPEN-SYM and CLOSE-SYM are respectively the
1196 symbols returned in open and close tokens."
1197 (let ((specs (cons spec1 specs
))
1198 spec open olist clist
)
1200 (setq spec
(car specs
)
1203 ;; build alist ((OPEN-DELIM OPEN-SYM BLOCK-SYM) ...)
1204 olist
(cons (list (car open
) (cadr open
) (car spec
)) olist
)
1205 ;; build alist ((CLOSE-DELIM CLOSE-SYM) ...)
1206 clist
(cons (nth 2 spec
) clist
)))
1207 `(define-lex-analyzer ,name
1210 (looking-at "\\(\\s(\\|\\s)\\)")
1211 (let ((text (match-string 0)) match
)
1213 ((setq match
(assoc text
',olist
))
1214 (if (or (not semantic-lex-maximum-depth
)
1215 (< semantic-lex-current-depth semantic-lex-maximum-depth
))
1217 (setq semantic-lex-current-depth
(1+ semantic-lex-current-depth
))
1218 (semantic-lex-push-token
1221 (match-beginning 0) (match-end 0))))
1222 (semantic-lex-push-token
1227 (semantic-lex-unterminated-syntax-protection (nth 2 match
)
1232 ((setq match
(assoc text
',clist
))
1233 (setq semantic-lex-current-depth
(1- semantic-lex-current-depth
))
1234 (semantic-lex-push-token
1237 (match-beginning 0) (match-end 0)))))))
1242 ;; Pre-defined common analyzers.
1244 (define-lex-analyzer semantic-lex-default-action
1245 "The default action when no other lexical actions match text.
1246 This action will just throw an error."
1248 (error "Unmatched Text during Lexical Analysis"))
1250 (define-lex-analyzer semantic-lex-beginning-of-line
1251 "Detect and create a beginning of line token (BOL)."
1253 ;; Just insert a (bol N . N) token in the token stream,
1254 ;; without moving the point. N is the point at the
1255 ;; beginning of line.
1256 (semantic-lex-push-token (semantic-lex-token 'bol
(point) (point)))
1258 ;; We identify and add the BOL token onto the stream, but since
1259 ;; semantic-lex-end-point doesn't move, we always fail CONDITION, and have no
1263 (define-lex-simple-regex-analyzer semantic-lex-newline
1264 "Detect and create newline tokens."
1265 "\\s-*\\(\n\\|\\s>\\)" 'newline
1)
1267 (define-lex-regex-analyzer semantic-lex-newline-as-whitespace
1268 "Detect and create newline tokens.
1269 Use this ONLY if newlines are not whitespace characters (such as when
1270 they are comment end characters) AND when you want whitespace tokens."
1271 "\\s-*\\(\n\\|\\s>\\)"
1272 ;; Language wants whitespaces. Create a token for it.
1273 (if (eq (semantic-lex-token-class (car semantic-lex-token-stream
))
1275 ;; Merge whitespace tokens together if they are adjacent. Two
1276 ;; whitespace tokens may be sperated by a comment which is not in
1277 ;; the token stream.
1278 (setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream
))
1280 (semantic-lex-push-token
1282 'whitespace
(match-beginning 0) (match-end 0)))))
1284 (define-lex-regex-analyzer semantic-lex-ignore-newline
1285 "Detect and ignore newline tokens.
1286 Use this ONLY if newlines are not whitespace characters (such as when
1287 they are comment end characters)."
1288 "\\s-*\\(\n\\|\\s>\\)"
1289 (setq semantic-lex-end-point
(match-end 0)))
1291 (define-lex-regex-analyzer semantic-lex-whitespace
1292 "Detect and create whitespace tokens."
1293 ;; catch whitespace when needed
1295 ;; Language wants whitespaces. Create a token for it.
1296 (if (eq (semantic-lex-token-class (car semantic-lex-token-stream
))
1298 ;; Merge whitespace tokens together if they are adjacent. Two
1299 ;; whitespace tokens may be sperated by a comment which is not in
1300 ;; the token stream.
1302 (setq semantic-lex-end-point
(match-end 0))
1303 (setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream
))
1304 semantic-lex-end-point
))
1305 (semantic-lex-push-token
1307 'whitespace
(match-beginning 0) (match-end 0)))))
1309 (define-lex-regex-analyzer semantic-lex-ignore-whitespace
1310 "Detect and skip over whitespace tokens."
1311 ;; catch whitespace when needed
1313 ;; Skip over the detected whitespace, do not create a token for it.
1314 (setq semantic-lex-end-point
(match-end 0)))
1316 (define-lex-simple-regex-analyzer semantic-lex-number
1317 "Detect and create number tokens.
1318 See `semantic-lex-number-expression' for details on matching numbers,
1319 and number formats."
1320 semantic-lex-number-expression
'number
)
1322 (define-lex-regex-analyzer semantic-lex-symbol-or-keyword
1323 "Detect and create symbol and keyword tokens."
1324 "\\(\\sw\\|\\s_\\)+"
1325 (semantic-lex-push-token
1327 (or (semantic-lex-keyword-p (match-string 0)) 'symbol
)
1328 (match-beginning 0) (match-end 0))))
1330 (define-lex-simple-regex-analyzer semantic-lex-charquote
1331 "Detect and create charquote tokens."
1332 ;; Character quoting characters (ie, \n as newline)
1333 "\\s\\+" 'charquote
)
1335 (define-lex-simple-regex-analyzer semantic-lex-punctuation
1336 "Detect and create punctuation tokens."
1337 "\\(\\s.\\|\\s$\\|\\s'\\)" 'punctuation
)
1339 (define-lex-analyzer semantic-lex-punctuation-type
1340 "Detect and create a punctuation type token.
1341 Recognized punctuations are defined in the current table of lexical
1342 types, as the value of the `punctuation' token type."
1343 (and (looking-at "\\(\\s.\\|\\s$\\|\\s'\\)+")
1344 (let* ((key (match-string 0))
1345 (pos (match-beginning 0))
1348 (lst (semantic-lex-type-value "punctuation" t
))
1349 (def (car lst
)) ;; default lexical symbol or nil
1350 (lst (cdr lst
)) ;; alist of (LEX-SYM . PUNCT-STRING)
1353 ;; Starting with the longest one, search if the
1354 ;; punctuation string is defined for this language.
1355 (while (and (> len
0) (not (setq elt
(rassoc key lst
))))
1357 key
(substring key
0 len
))))
1358 (if elt
;; Return the punctuation token found
1359 (semantic-lex-push-token
1360 (semantic-lex-token (car elt
) pos
(+ pos len
)))
1361 (if def
;; Return a default generic token
1362 (semantic-lex-push-token
1363 (semantic-lex-token def pos end
))
1367 (define-lex-regex-analyzer semantic-lex-paren-or-list
1368 "Detect open parenthesis.
1369 Return either a paren token or a semantic list token depending on
1370 `semantic-lex-current-depth'."
1372 (if (or (not semantic-lex-maximum-depth
)
1373 (< semantic-lex-current-depth semantic-lex-maximum-depth
))
1375 (setq semantic-lex-current-depth
(1+ semantic-lex-current-depth
))
1376 (semantic-lex-push-token
1378 'open-paren
(match-beginning 0) (match-end 0))))
1379 (semantic-lex-push-token
1381 'semantic-list
(match-beginning 0)
1383 (semantic-lex-unterminated-syntax-protection 'semantic-list
1389 (define-lex-simple-regex-analyzer semantic-lex-open-paren
1390 "Detect and create an open parenthisis token."
1391 "\\s(" 'open-paren
0 (setq semantic-lex-current-depth
(1+ semantic-lex-current-depth
)))
1393 (define-lex-simple-regex-analyzer semantic-lex-close-paren
1394 "Detect and create a close paren token."
1395 "\\s)" 'close-paren
0 (setq semantic-lex-current-depth
(1- semantic-lex-current-depth
)))
1397 (define-lex-regex-analyzer semantic-lex-string
1398 "Detect and create a string token."
1400 ;; Zing to the end of this string.
1401 (semantic-lex-push-token
1405 (semantic-lex-unterminated-syntax-protection 'string
1410 (define-lex-regex-analyzer semantic-lex-comments
1411 "Detect and create a comment token."
1412 semantic-lex-comment-regex
1415 ;; Generate newline token if enabled
1416 (if (bolp) (backward-char 1))
1417 (setq semantic-lex-end-point
(point))
1418 ;; Language wants comments or want them as whitespaces,
1419 ;; link them together.
1420 (if (eq (semantic-lex-token-class (car semantic-lex-token-stream
)) 'comment
)
1421 (setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream
))
1422 semantic-lex-end-point
)
1423 (semantic-lex-push-token
1425 'comment
(match-beginning 0) semantic-lex-end-point
)))))
1427 (define-lex-regex-analyzer semantic-lex-comments-as-whitespace
1428 "Detect comments and create a whitespace token."
1429 semantic-lex-comment-regex
1432 ;; Generate newline token if enabled
1433 (if (bolp) (backward-char 1))
1434 (setq semantic-lex-end-point
(point))
1435 ;; Language wants comments or want them as whitespaces,
1436 ;; link them together.
1437 (if (eq (semantic-lex-token-class (car semantic-lex-token-stream
)) 'whitespace
)
1438 (setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream
))
1439 semantic-lex-end-point
)
1440 (semantic-lex-push-token
1442 'whitespace
(match-beginning 0) semantic-lex-end-point
)))))
1444 (define-lex-regex-analyzer semantic-lex-ignore-comments
1445 "Detect and create a comment token."
1446 semantic-lex-comment-regex
1447 (let ((comment-start-point (point)))
1449 (if (eq (point) comment-start-point
)
1450 ;; In this case our start-skip string failed
1451 ;; to work properly. Lets try and move over
1452 ;; whatever white space we matched to begin
1454 (skip-syntax-forward "-.'"
1458 ;; We may need to back up so newlines or whitespace is generated.
1461 (if (eq (point) comment-start-point
)
1462 (error "Strange comment syntax prevents lexical analysis"))
1463 (setq semantic-lex-end-point
(point))))
1467 ;; Predefined lexers that could be used instead of creating new
1470 (define-lex semantic-comment-lexer
1471 "A simple lexical analyzer that handles comments.
1472 This lexer will only return comment tokens. It is the default lexer
1473 used by `semantic-find-doc-snarf-comment' to snarf up the comment at
1475 semantic-lex-ignore-whitespace
1476 semantic-lex-ignore-newline
1477 semantic-lex-comments
1478 semantic-lex-default-action
)
1482 (define-lex semantic-simple-lexer
1483 "A simple lexical analyzer that handles simple buffers.
1484 This lexer ignores comments and whitespace, and will return
1485 syntax as specified by the syntax table."
1486 semantic-lex-ignore-whitespace
1487 semantic-lex-ignore-newline
1489 semantic-lex-symbol-or-keyword
1490 semantic-lex-charquote
1491 semantic-lex-paren-or-list
1492 semantic-lex-close-paren
1494 semantic-lex-ignore-comments
1495 semantic-lex-punctuation
1496 semantic-lex-default-action
)
1498 ;;; Analyzers generated from grammar.
1500 ;; Some analyzers are hand written. Analyzers created with these
1501 ;; functions are generated from the grammar files.
1503 (defmacro define-lex-keyword-type-analyzer
(name doc syntax
)
1504 "Define a keyword type analyzer NAME with DOC string.
1505 SYNTAX is the regexp that matches a keyword syntactic expression."
1506 (let ((key (make-symbol "key")))
1507 `(define-lex-analyzer ,name
1509 (and (looking-at ,syntax
)
1510 (let ((,key
(semantic-lex-keyword-p (match-string 0))))
1512 (semantic-lex-push-token
1514 ,key
(match-beginning 0) (match-end 0)))))))
1517 (defmacro define-lex-sexp-type-analyzer
(name doc syntax token
)
1518 "Define a sexp type analyzer NAME with DOC string.
1519 SYNTAX is the regexp that matches the beginning of the s-expression.
1520 TOKEN is the lexical token returned when SYNTAX matches."
1521 `(define-lex-regex-analyzer ,name
1524 (semantic-lex-push-token
1528 (semantic-lex-unterminated-syntax-protection ,token
1533 (defmacro define-lex-regex-type-analyzer
(name doc syntax matches default
)
1534 "Define a regexp type analyzer NAME with DOC string.
1535 SYNTAX is the regexp that matches a syntactic expression.
1536 MATCHES is an alist of lexical elements used to refine the syntactic
1538 DEFAULT is the default lexical token returned when no MATCHES."
1540 (let* ((val (make-symbol "val"))
1541 (lst (make-symbol "lst"))
1542 (elt (make-symbol "elt"))
1543 (pos (make-symbol "pos"))
1544 (end (make-symbol "end")))
1545 `(define-lex-analyzer ,name
1547 (and (looking-at ,syntax
)
1548 (let* ((,val
(match-string 0))
1549 (,pos
(match-beginning 0))
1550 (,end
(match-end 0))
1553 (while (and ,lst
(not ,elt
))
1554 (if (string-match (cdar ,lst
) ,val
)
1555 (setq ,elt
(caar ,lst
))
1556 (setq ,lst
(cdr ,lst
))))
1557 (semantic-lex-push-token
1558 (semantic-lex-token (or ,elt
,default
) ,pos
,end
))))
1560 `(define-lex-simple-regex-analyzer ,name
1565 (defmacro define-lex-string-type-analyzer
(name doc syntax matches default
)
1566 "Define a string type analyzer NAME with DOC string.
1567 SYNTAX is the regexp that matches a syntactic expression.
1568 MATCHES is an alist of lexical elements used to refine the syntactic
1570 DEFAULT is the default lexical token returned when no MATCHES."
1572 (let* ((val (make-symbol "val"))
1573 (lst (make-symbol "lst"))
1574 (elt (make-symbol "elt"))
1575 (pos (make-symbol "pos"))
1576 (end (make-symbol "end"))
1577 (len (make-symbol "len")))
1578 `(define-lex-analyzer ,name
1580 (and (looking-at ,syntax
)
1581 (let* ((,val
(match-string 0))
1582 (,pos
(match-beginning 0))
1583 (,end
(match-end 0))
1584 (,len
(- ,end
,pos
))
1587 ;; Starting with the longest one, search if a lexical
1588 ;; value match a token defined for this language.
1589 (while (and (> ,len
0) (not (setq ,elt
(rassoc ,val
,lst
))))
1590 (setq ,len
(1- ,len
)
1591 ,val
(substring ,val
0 ,len
)))
1592 (when ,elt
;; Adjust token end position.
1593 (setq ,elt
(car ,elt
)
1594 ,end
(+ ,pos
,len
)))
1595 (semantic-lex-push-token
1596 (semantic-lex-token (or ,elt
,default
) ,pos
,end
))))
1598 `(define-lex-simple-regex-analyzer ,name
1603 (defmacro define-lex-block-type-analyzer
(name doc syntax matches
)
1604 "Define a block type analyzer NAME with DOC string.
1606 SYNTAX is the regexp that matches block delimiters, typically the
1607 open (`\\\\s(') and close (`\\\\s)') parenthesis syntax classes.
1609 MATCHES is a pair (OPEN-SPECS . CLOSE-SPECS) that defines blocks.
1611 OPEN-SPECS is a list of (OPEN-DELIM OPEN-TOKEN BLOCK-TOKEN) elements
1614 OPEN-DELIM is a string: the block open delimiter character.
1616 OPEN-TOKEN is the lexical token class associated to the OPEN-DELIM
1619 BLOCK-TOKEN is the lexical token class associated to the block
1620 that starts at the OPEN-DELIM delimiter.
1622 CLOSE-SPECS is a list of (CLOSE-DELIM CLOSE-TOKEN) elements where:
1624 CLOSE-DELIM is a string: the block end delimiter character.
1626 CLOSE-TOKEN is the lexical token class associated to the
1627 CLOSE-DELIM delimiter.
1629 Each element in OPEN-SPECS must have a corresponding element in
1632 The lexer will return a BLOCK-TOKEN token when the value of
1633 `semantic-lex-current-depth' is greater than or equal to the maximum
1634 depth of parenthesis tracking (see also the function `semantic-lex').
1635 Otherwise it will return OPEN-TOKEN and CLOSE-TOKEN tokens.
1637 TO DO: Put the following in the developer's guide and just put a
1642 The value of a block token must be a string that contains a readable
1645 \"(OPEN-TOKEN CLOSE-TOKEN)\"
1647 OPEN-TOKEN and CLOSE-TOKEN represent the block delimiters, and must be
1648 lexical tokens of respectively `open-paren' and `close-paren' types.
1649 Their value is the corresponding delimiter character as a string.
1651 Here is a small example to analyze a parenthesis block:
1653 %token <block> PAREN_BLOCK \"(LPAREN RPAREN)\"
1654 %token <open-paren> LPAREN \"(\"
1655 %token <close-paren> RPAREN \")\"
1657 When the lexer encounters the open-paren delimiter \"(\":
1659 - If the maximum depth of parenthesis tracking is not reached (that
1660 is, current depth < max depth), it returns a (LPAREN start . end)
1661 token, then continue analysis inside the block. Later, when the
1662 corresponding close-paren delimiter \")\" will be encountered, it
1663 will return a (RPAREN start . end) token.
1665 - If the maximum depth of parenthesis tracking is reached (current
1666 depth >= max depth), it returns the whole parenthesis block as
1667 a (PAREN_BLOCK start . end) token."
1668 (let* ((val (make-symbol "val"))
1669 (lst (make-symbol "lst"))
1670 (elt (make-symbol "elt")))
1671 `(define-lex-analyzer ,name
1674 (looking-at ,syntax
) ;; "\\(\\s(\\|\\s)\\)"
1675 (let ((,val
(match-string 0))
1679 ((setq ,elt
(assoc ,val
(car ,lst
)))
1680 (if (or (not semantic-lex-maximum-depth
)
1681 (< semantic-lex-current-depth semantic-lex-maximum-depth
))
1683 (setq semantic-lex-current-depth
(1+ semantic-lex-current-depth
))
1684 (semantic-lex-push-token
1687 (match-beginning 0) (match-end 0))))
1688 (semantic-lex-push-token
1693 (semantic-lex-unterminated-syntax-protection (nth 2 ,elt
)
1696 ((setq ,elt
(assoc ,val
(cdr ,lst
)))
1697 (setq semantic-lex-current-depth
(1- semantic-lex-current-depth
))
1698 (semantic-lex-push-token
1701 (match-beginning 0) (match-end 0))))
1707 ;; The semantic lexers, unlike other lexers, can throw errors on
1708 ;; unbalanced syntax. Since editing is all about changeging test
1709 ;; we need to provide a convenient way to protect against syntactic
1712 (defmacro semantic-lex-catch-errors
(symbol &rest forms
)
1713 "Using SYMBOL, execute FORMS catching lexical errors.
1714 If FORMS results in a call to the parser that throws a lexical error,
1715 the error will be caught here without the buffer's cache being thrown
1717 If there is an error, the syntax that failed is returned.
1718 If there is no error, then the last value of FORMS is returned."
1719 (let ((ret (make-symbol "ret"))
1720 (syntax (make-symbol "syntax"))
1721 (start (make-symbol "start"))
1722 (end (make-symbol "end")))
1723 `(let* ((semantic-lex-unterminated-syntax-end-function
1724 (lambda (,syntax
,start
,end
)
1725 (throw ',symbol
,syntax
)))
1726 ;; Delete the below when semantic-flex is fully retired.
1727 (semantic-flex-unterminated-syntax-end-function
1728 semantic-lex-unterminated-syntax-end-function
)
1729 (,ret
(catch ',symbol
1733 ;; Great Sadness. Assume that FORMS execute within the
1734 ;; confines of the current buffer only! Mark this thing
1735 ;; unparseable iff the special symbol was thrown. This
1736 ;; will prevent future calls from parsing, but will allow
1737 ;; then to still return the cache.
1739 ;; Leave this message off. If an APP using this fcn wants
1740 ;; a message, they can do it themselves. This cleans up
1741 ;; problems with the idle scheduler obscuring useful data.
1742 ;;(message "Buffer not currently parsable (%S)." ,ret)
1743 (semantic-parse-tree-unparseable))
1745 (put 'semantic-lex-catch-errors
'lisp-indent-function
1)
1748 ;;; Interfacing with edebug
1754 (def-edebug-spec define-lex
1755 (&define name stringp
(&rest symbolp
))
1757 (def-edebug-spec define-lex-analyzer
1758 (&define name stringp form def-body
)
1760 (def-edebug-spec define-lex-regex-analyzer
1761 (&define name stringp form def-body
)
1763 (def-edebug-spec define-lex-simple-regex-analyzer
1764 (&define name stringp form symbolp
[ &optional form
] def-body
)
1766 (def-edebug-spec define-lex-block-analyzer
1767 (&define name stringp form
(&rest form
))
1769 (def-edebug-spec semantic-lex-catch-errors
1775 ;;; Compatibility with Semantic 1.x lexical analysis
1777 ;; NOTE: DELETE THIS SOMEDAY SOON
1779 (semantic-alias-obsolete 'semantic-flex-start
'semantic-lex-token-start
)
1780 (semantic-alias-obsolete 'semantic-flex-end
'semantic-lex-token-end
)
1781 (semantic-alias-obsolete 'semantic-flex-text
'semantic-lex-token-text
)
1782 (semantic-alias-obsolete 'semantic-flex-make-keyword-table
'semantic-lex-make-keyword-table
)
1783 (semantic-alias-obsolete 'semantic-flex-keyword-p
'semantic-lex-keyword-p
)
1784 (semantic-alias-obsolete 'semantic-flex-keyword-put
'semantic-lex-keyword-put
)
1785 (semantic-alias-obsolete 'semantic-flex-keyword-get
'semantic-lex-keyword-get
)
1786 (semantic-alias-obsolete 'semantic-flex-map-keywords
'semantic-lex-map-keywords
)
1787 (semantic-alias-obsolete 'semantic-flex-keywords
'semantic-lex-keywords
)
1788 (semantic-alias-obsolete 'semantic-flex-buffer
'semantic-lex-buffer
)
1789 (semantic-alias-obsolete 'semantic-flex-list
'semantic-lex-list
)
1791 ;; This simple scanner uses the syntax table to generate a stream of
1792 ;; simple tokens of the form:
1794 ;; (SYMBOL START . END)
1796 ;; Where symbol is the type of thing it is. START and END mark that
1797 ;; objects boundary.
1799 (defvar semantic-flex-tokens semantic-lex-tokens
1800 "An alist of of semantic token types.
1801 See variable `semantic-lex-tokens'.")
1803 (defvar semantic-flex-unterminated-syntax-end-function
1804 (lambda (syntax syntax-start flex-end
) flex-end
)
1805 "Function called when unterminated syntax is encountered.
1806 This should be set to one function. That function should take three
1807 parameters. The SYNTAX, or type of syntax which is unterminated.
1808 SYNTAX-START where the broken syntax begins.
1809 FLEX-END is where the lexical analysis was asked to end.
1810 This function can be used for languages that can intelligently fix up
1811 broken syntax, or the exit lexical analysis via `throw' or `signal'
1812 when finding unterminated syntax.")
1814 (defvar semantic-flex-extensions nil
1815 "Buffer local extensions to the lexical analyzer.
1816 This should contain an alist with a key of a regex and a data element of
1817 a function. The function should both move point, and return a lexical
1820 nil is also a valid return value.
1821 TYPE can be any type of symbol, as long as it doesn't occur as a
1822 nonterminal in the language definition.")
1823 (make-variable-buffer-local 'semantic-flex-extensions
)
1825 (defvar semantic-flex-syntax-modifications nil
1826 "Changes to the syntax table for this buffer.
1827 These changes are active only while the buffer is being flexed.
1828 This is a list where each element has the form:
1830 CHAR is the char passed to `modify-syntax-entry',
1831 and CLASS is the string also passed to `modify-syntax-entry' to define
1832 what syntax class CHAR has.")
1833 (make-variable-buffer-local 'semantic-flex-syntax-modifications
)
1835 (defvar semantic-ignore-comments t
1836 "Default comment handling.
1837 t means to strip comments when flexing. Nil means to keep comments
1838 as part of the token stream.")
1839 (make-variable-buffer-local 'semantic-ignore-comments
)
1841 (defvar semantic-flex-enable-newlines nil
1842 "When flexing, report 'newlines as syntactic elements.
1843 Useful for languages where the newline is a special case terminator.
1844 Only set this on a per mode basis, not globally.")
1845 (make-variable-buffer-local 'semantic-flex-enable-newlines
)
1847 (defvar semantic-flex-enable-whitespace nil
1848 "When flexing, report 'whitespace as syntactic elements.
1849 Useful for languages where the syntax is whitespace dependent.
1850 Only set this on a per mode basis, not globally.")
1851 (make-variable-buffer-local 'semantic-flex-enable-whitespace
)
1853 (defvar semantic-flex-enable-bol nil
1854 "When flexing, report beginning of lines as syntactic elements.
1855 Useful for languages like python which are indentation sensitive.
1856 Only set this on a per mode basis, not globally.")
1857 (make-variable-buffer-local 'semantic-flex-enable-bol
)
1859 (defvar semantic-number-expression semantic-lex-number-expression
1860 "See variable `semantic-lex-number-expression'.")
1861 (make-variable-buffer-local 'semantic-number-expression
)
1863 (defvar semantic-flex-depth
0
1864 "Default flexing depth.
1865 This specifies how many lists to create tokens in.")
1866 (make-variable-buffer-local 'semantic-flex-depth
)
1868 (defun semantic-flex (start end
&optional depth length
)
1869 "Using the syntax table, do something roughly equivalent to flex.
1870 Semantically check between START and END. Optional argument DEPTH
1871 indicates at what level to scan over entire lists.
1872 The return value is a token stream. Each element is a list, such of
1873 the form (symbol start-expression . end-expression) where SYMBOL
1874 denotes the token type.
1875 See `semantic-flex-tokens' variable for details on token types.
1876 END does not mark the end of the text scanned, only the end of the
1877 beginning of text scanned. Thus, if a string extends past END, the
1878 end of the return token will be larger than END. To truly restrict
1879 scanning, use `narrow-to-region'.
1880 The last argument, LENGTH specifies that `semantic-flex' should only
1881 return LENGTH tokens."
1882 (message "`semantic-flex' is an obsolete function. Use `define-lex' to create lexers.")
1883 (if (not semantic-flex-keywords-obarray
)
1884 (setq semantic-flex-keywords-obarray
[ nil
]))
1889 (cs (if comment-start-skip
1890 (concat "\\(\\s<\\|" comment-start-skip
"\\)")
1891 (concat "\\(\\s<\\)")))
1892 (newsyntax (copy-syntax-table (syntax-table)))
1893 (mods semantic-flex-syntax-modifications
)
1894 ;; Use the default depth if it is not specified.
1895 (depth (or depth semantic-flex-depth
)))
1896 ;; Update the syntax table
1898 (modify-syntax-entry (car (car mods
)) (car (cdr (car mods
))) newsyntax
)
1899 (setq mods
(cdr mods
)))
1900 (with-syntax-table newsyntax
1902 (while (and (< (point) end
) (or (not length
) (<= (length ts
) length
)))
1904 ;; catch beginning of lines when needed.
1905 ;; Must be done before catching any other tokens!
1906 ((and semantic-flex-enable-bol
1908 ;; Just insert a (bol N . N) token in the token stream,
1909 ;; without moving the point. N is the point at the
1910 ;; beginning of line.
1911 (setq ts
(cons (cons 'bol
(cons (point) (point))) ts
))
1913 ;; special extensions, includes whitespace, nl, etc.
1914 ((and semantic-flex-extensions
1915 (let ((fe semantic-flex-extensions
)
1918 (if (looking-at (car (car fe
)))
1919 (setq ts
(cons (funcall (cdr (car fe
))) ts
)
1924 (if (and r
(not (car ts
))) (setq ts
(cdr ts
)))
1926 ;; catch newlines when needed
1927 ((looking-at "\\s-*\\(\n\\|\\s>\\)")
1928 (if semantic-flex-enable-newlines
1929 (setq ep
(match-end 1)
1930 ts
(cons (cons 'newline
1931 (cons (match-beginning 1) ep
))
1933 ;; catch whitespace when needed
1934 ((looking-at "\\s-+")
1935 (if semantic-flex-enable-whitespace
1936 ;; Language wants whitespaces, link them together.
1937 (if (eq (car (car ts
)) 'whitespace
)
1938 (setcdr (cdr (car ts
)) (match-end 0))
1939 (setq ts
(cons (cons 'whitespace
1940 (cons (match-beginning 0)
1944 ((and semantic-number-expression
1945 (looking-at semantic-number-expression
))
1946 (setq ts
(cons (cons 'number
1947 (cons (match-beginning 0)
1951 ((looking-at "\\(\\sw\\|\\s_\\)+")
1952 (setq ts
(cons (cons
1953 ;; Get info on if this is a keyword or not
1954 (or (semantic-lex-keyword-p (match-string 0))
1956 (cons (match-beginning 0) (match-end 0)))
1958 ;; Character quoting characters (ie, \n as newline)
1959 ((looking-at "\\s\\+")
1960 (setq ts
(cons (cons 'charquote
1961 (cons (match-beginning 0) (match-end 0)))
1963 ;; Open parens, or semantic-lists.
1964 ((looking-at "\\s(")
1965 (if (or (not depth
) (< curdepth depth
))
1967 (setq curdepth
(1+ curdepth
))
1968 (setq ts
(cons (cons 'open-paren
1969 (cons (match-beginning 0) (match-end 0)))
1972 (cons 'semantic-list
1973 (cons (match-beginning 0)
1977 ;; This case makes flex robust
1982 semantic-flex-unterminated-syntax-end-function
1985 (setq ep
(point)))))
1988 ((looking-at "\\s)")
1989 (setq ts
(cons (cons 'close-paren
1990 (cons (match-beginning 0) (match-end 0)))
1992 (setq curdepth
(1- curdepth
)))
1993 ;; String initiators
1994 ((looking-at "\\s\"")
1995 ;; Zing to the end of this string.
1996 (setq ts
(cons (cons 'string
1997 (cons (match-beginning 0)
2001 ;; This case makes flex
2002 ;; robust to broken strings.
2006 semantic-flex-unterminated-syntax-end-function
2009 (setq ep
(point)))))
2013 (if (and semantic-ignore-comments
2014 (not semantic-flex-enable-whitespace
))
2015 ;; If the language doesn't deal with comments nor
2016 ;; whitespaces, ignore them here.
2017 (let ((comment-start-point (point)))
2019 (if (eq (point) comment-start-point
)
2020 ;; In this case our start-skip string failed
2021 ;; to work properly. Lets try and move over
2022 ;; whatever white space we matched to begin
2024 (skip-syntax-forward "-.'"
2028 ;;(forward-comment 1)
2029 ;; Generate newline token if enabled
2030 (if (and semantic-flex-enable-newlines
2033 (if (eq (point) comment-start-point
)
2034 (error "Strange comment syntax prevents lexical analysis"))
2036 (let ((tk (if semantic-ignore-comments
'whitespace
'comment
)))
2039 ;; Generate newline token if enabled
2040 (if (and semantic-flex-enable-newlines
2044 ;; Language wants comments or want them as whitespaces,
2045 ;; link them together.
2046 (if (eq (car (car ts
)) tk
)
2047 (setcdr (cdr (car ts
)) ep
)
2048 (setq ts
(cons (cons tk
(cons (match-beginning 0) ep
))
2051 ((looking-at "\\(\\s.\\|\\s$\\|\\s'\\)")
2052 (setq ts
(cons (cons 'punctuation
2053 (cons (match-beginning 0) (match-end 0)))
2057 (error "What is that?")))
2058 (goto-char (or ep
(match-end 0)))
2060 ;; maybe catch the last beginning of line when needed
2061 (and semantic-flex-enable-bol
2064 (setq ts
(cons (cons 'bol
(cons (point) (point))) ts
)))
2066 ;;(message "Flexing muscles...done")
2069 (provide 'semantic
/lex
)
2072 ;; generated-autoload-file: "loaddefs.el"
2073 ;; generated-autoload-feature: semantic/loaddefs
2074 ;; generated-autoload-load-name: "semantic/lex"
2077 ;;; semantic-lex.el ends here