1 ;;; semantic/lex.el --- Lexical Analyzer builder
3 ;; Copyright (C) 1999-2012 Free Software Foundation, Inc.
5 ;; Author: Eric M. Ludlam <zappo@gnu.org>
7 ;; This file is part of GNU Emacs.
9 ;; GNU Emacs is free software: you can redistribute it and/or modify
10 ;; it under the terms of the GNU General Public License as published by
11 ;; the Free Software Foundation, either version 3 of the License, or
12 ;; (at your option) any later version.
14 ;; GNU Emacs is distributed in the hope that it will be useful,
15 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
16 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 ;; GNU General Public License for more details.
19 ;; You should have received a copy of the GNU General Public License
20 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
24 ;; This file handles the creation of lexical analyzers for different
25 ;; languages in Emacs Lisp. The purpose of a lexical analyzer is to
26 ;; convert a buffer into a list of lexical tokens. Each token
27 ;; contains the token class (such as 'number, 'symbol, 'IF, etc) and
28 ;; the location in the buffer it was found. Optionally, a token also
29 ;; contains a string representing what is at the designated buffer
32 ;; Tokens are pushed onto a token stream, which is basically a list of
33 ;; all the lexical tokens from the analyzed region. The token stream
34 ;; is then handed to the grammar which parsers the file.
38 ;; Each analyzer specifies a condition and forms. These conditions
39 ;; and forms are assembled into a function by `define-lex' that does
40 ;; the lexical analysis.
42 ;; In the lexical analyzer created with `define-lex', each condition
43 ;; is tested for a given point. When the condition is true, the forms
46 ;; The forms can push a lexical token onto the token stream. The
47 ;; analyzer forms also must move the current analyzer point. If the
48 ;; analyzer point is moved without pushing a token, then the matched
49 ;; syntax is effectively ignored, or skipped.
51 ;; Thus, starting at the beginning of a region to be analyzed, each
52 ;; condition is tested. One will match, and a lexical token might be
53 ;; pushed, and the point is moved to the end of the lexical token
54 ;; identified. At the new position, the process occurs again until
55 ;; the end of the specified region is reached.
57 ;;; How to use semantic-lex
59 ;; To create a lexer for a language, use the `define-lex' macro.
61 ;; The `define-lex' macro accepts a list of lexical analyzers. Each
62 ;; analyzer is created with `define-lex-analyzer', or one of the
63 ;; derivative macros. A single analyzer defines a regular expression
64 ;; to match text in a buffer, and a short segment of code to create
67 ;; Each analyzer has a NAME, DOC, a CONDITION, and possibly some
68 ;; FORMS. The NAME is the name used in `define-lex'. The DOC
69 ;; describes what the analyzer should do.
71 ;; The CONDITION evaluates the text at the current point in the
72 ;; current buffer. If CONDITION is true, then the FORMS will be
75 ;; The purpose of the FORMS is to push new lexical tokens onto the
76 ;; list of tokens for the current buffer, and to move point after the
79 ;; Some macros for creating one analyzer are:
81 ;; define-lex-analyzer - A generic analyzer associating any style of
82 ;; condition to forms.
83 ;; define-lex-regex-analyzer - Matches a regular expression.
84 ;; define-lex-simple-regex-analyzer - Matches a regular expressions,
85 ;; and pushes the match.
86 ;; define-lex-block-analyzer - Matches list syntax, and defines
87 ;; handles open/close delimiters.
89 ;; These macros are used by the grammar compiler when lexical
90 ;; information is specified in a grammar:
91 ;; define-lex- * -type-analyzer - Matches syntax specified in
92 ;; a grammar, and pushes one token for it. The * would
93 ;; be `sexp' for things like lists or strings, and
94 ;; `string' for things that need to match some special
95 ;; string, such as "\\." where a literal match is needed.
99 ;; There are tables of different symbols managed in semantic-lex.el.
102 ;; Lexical keyword table - A Table of symbols declared in a grammar
103 ;; file with the %keyword declaration.
104 ;; Keywords are used by `semantic-lex-symbol-or-keyword'
105 ;; to create lexical tokens based on the keyword.
107 ;; Lexical type table - A table of symbols declared in a grammar
108 ;; file with the %type declaration.
109 ;; The grammar compiler uses the type table to create new
110 ;; lexical analyzers. These analyzers are then used to when
111 ;; a new lexical analyzer is made for a language.
115 ;; A lexical type defines a kind of lexical analyzer that will be
116 ;; automatically generated from a grammar file based on some
117 ;; predetermined attributes. For now these two attributes are
120 ;; * matchdatatype : define the kind of lexical analyzer. That is :
122 ;; - regexp : define a regexp analyzer (see
123 ;; `define-lex-regex-type-analyzer')
125 ;; - string : define a string analyzer (see
126 ;; `define-lex-string-type-analyzer')
128 ;; - block : define a block type analyzer (see
129 ;; `define-lex-block-type-analyzer')
131 ;; - sexp : define a sexp analyzer (see
132 ;; `define-lex-sexp-type-analyzer')
134 ;; - keyword : define a keyword analyzer (see
135 ;; `define-lex-keyword-type-analyzer')
137 ;; * syntax : define the syntax that matches a syntactic
138 ;; expression. When syntax is matched the corresponding type
139 ;; analyzer is entered and the resulting match data will be
140 ;; interpreted based on the kind of analyzer (see matchdatatype
143 ;; The following lexical types are predefined :
145 ;; +-------------+---------------+--------------------------------+
146 ;; | type | matchdatatype | syntax |
147 ;; +-------------+---------------+--------------------------------+
148 ;; | punctuation | string | "\\(\\s.\\|\\s$\\|\\s'\\)+" |
149 ;; | keyword | keyword | "\\(\\sw\\|\\s_\\)+" |
150 ;; | symbol | regexp | "\\(\\sw\\|\\s_\\)+" |
151 ;; | string | sexp | "\\s\"" |
152 ;; | number | regexp | semantic-lex-number-expression |
153 ;; | block | block | "\\s(\\|\\s)" |
154 ;; +-------------+---------------+--------------------------------+
156 ;; In a grammar you must use a %type expression to automatically generate
157 ;; the corresponding analyzers of that type.
159 ;; Here is an example to auto-generate punctuation analyzers
160 ;; with 'matchdatatype and 'syntax predefined (see table above)
162 ;; %type <punctuation> ;; will auto-generate this kind of analyzers
164 ;; It is equivalent to write :
166 ;; %type <punctuation> syntax "\\(\\s.\\|\\s$\\|\\s'\\)+" matchdatatype string
168 ;; ;; Some punctuation based on the type defines above
170 ;; %token <punctuation> NOT "!"
171 ;; %token <punctuation> NOTEQ "!="
172 ;; %token <punctuation> MOD "%"
173 ;; %token <punctuation> MODEQ "%="
176 ;;; On the Semantic 1.x lexer
178 ;; In semantic 1.x, the lexical analyzer was an all purpose routine.
179 ;; To boost efficiency, the analyzer is now a series of routines that
180 ;; are constructed at build time into a single routine. This will
181 ;; eliminate unneeded if statements to speed the lexer.
183 (require 'semantic
/fw
)
187 ;;; Semantic 2.x lexical analysis
189 (defun semantic-lex-map-symbols (fun table
&optional property
)
190 "Call function FUN on every symbol in TABLE.
191 If optional PROPERTY is non-nil, call FUN only on every symbol which
192 as a PROPERTY value. FUN receives a symbol as argument."
196 (if (or (null property
) (get symbol property
))
197 (funcall fun symbol
)))
200 ;;; Lexical keyword table handling.
202 ;; These keywords are keywords defined for using in a grammar with the
203 ;; %keyword declaration, and are not keywords used in Emacs Lisp.
205 (defvar semantic-flex-keywords-obarray nil
206 "Buffer local keyword obarray for the lexical analyzer.
207 These keywords are matched explicitly, and converted into special symbols.")
208 (make-variable-buffer-local 'semantic-flex-keywords-obarray
)
210 (defmacro semantic-lex-keyword-invalid
(name)
211 "Signal that NAME is an invalid keyword name."
212 `(signal 'wrong-type-argument
'(semantic-lex-keyword-p ,name
)))
214 (defsubst semantic-lex-keyword-symbol
(name)
215 "Return keyword symbol with NAME or nil if not found."
216 (and (arrayp semantic-flex-keywords-obarray
)
218 (intern-soft name semantic-flex-keywords-obarray
)))
220 (defsubst semantic-lex-keyword-p
(name)
221 "Return non-nil if a keyword with NAME exists in the keyword table.
222 Return nil otherwise."
223 (and (setq name
(semantic-lex-keyword-symbol name
))
224 (symbol-value name
)))
226 (defsubst semantic-lex-keyword-set
(name value
)
227 "Set value of keyword with NAME to VALUE and return VALUE."
228 (set (intern name semantic-flex-keywords-obarray
) value
))
230 (defsubst semantic-lex-keyword-value
(name)
231 "Return value of keyword with NAME.
232 Signal an error if a keyword with NAME does not exist."
233 (let ((keyword (semantic-lex-keyword-symbol name
)))
235 (symbol-value keyword
)
236 (semantic-lex-keyword-invalid name
))))
238 (defsubst semantic-lex-keyword-put
(name property value
)
239 "For keyword with NAME, set its PROPERTY to VALUE."
240 (let ((keyword (semantic-lex-keyword-symbol name
)))
242 (put keyword property value
)
243 (semantic-lex-keyword-invalid name
))))
245 (defsubst semantic-lex-keyword-get
(name property
)
246 "For keyword with NAME, return its PROPERTY value."
247 (let ((keyword (semantic-lex-keyword-symbol name
)))
249 (get keyword property
)
250 (semantic-lex-keyword-invalid name
))))
252 (defun semantic-lex-make-keyword-table (specs &optional propspecs
)
253 "Convert keyword SPECS into an obarray and return it.
254 SPECS must be a list of (NAME . TOKSYM) elements, where:
256 NAME is the name of the keyword symbol to define.
257 TOKSYM is the lexical token symbol of that keyword.
259 If optional argument PROPSPECS is non nil, then interpret it, and
260 apply those properties.
261 PROPSPECS must be a list of (NAME PROPERTY VALUE) elements."
262 ;; Create the symbol hash table
263 (let ((semantic-flex-keywords-obarray (make-vector 13 0))
265 ;; fill it with stuff
267 (setq spec
(car specs
)
269 (semantic-lex-keyword-set (car spec
) (cdr spec
)))
270 ;; Apply all properties
272 (setq spec
(car propspecs
)
273 propspecs
(cdr propspecs
))
274 (semantic-lex-keyword-put (car spec
) (nth 1 spec
) (nth 2 spec
)))
275 semantic-flex-keywords-obarray
))
277 (defsubst semantic-lex-map-keywords
(fun &optional property
)
278 "Call function FUN on every lexical keyword.
279 If optional PROPERTY is non-nil, call FUN only on every keyword which
280 as a PROPERTY value. FUN receives a lexical keyword as argument."
281 (semantic-lex-map-symbols
282 fun semantic-flex-keywords-obarray property
))
284 (defun semantic-lex-keywords (&optional property
)
285 "Return a list of lexical keywords.
286 If optional PROPERTY is non-nil, return only keywords which have a
289 (semantic-lex-map-keywords
290 #'(lambda (symbol) (setq keywords
(cons symbol keywords
)))
294 ;;; Inline functions:
296 (defvar semantic-lex-unterminated-syntax-end-function
)
297 (defvar semantic-lex-analysis-bounds
)
298 (defvar semantic-lex-end-point
)
300 (defsubst semantic-lex-token-bounds
(token)
301 "Fetch the start and end locations of the lexical token TOKEN.
302 Return a pair (START . END)."
303 (if (not (numberp (car (cdr token
))))
307 (defsubst semantic-lex-token-start
(token)
308 "Fetch the start position of the lexical token TOKEN.
309 See also the function `semantic-lex-token'."
310 (car (semantic-lex-token-bounds token
)))
312 (defsubst semantic-lex-token-end
(token)
313 "Fetch the end position of the lexical token TOKEN.
314 See also the function `semantic-lex-token'."
315 (cdr (semantic-lex-token-bounds token
)))
317 (defsubst semantic-lex-unterminated-syntax-detected
(syntax)
318 "Inside a lexical analyzer, use this when unterminated syntax was found.
319 Argument SYNTAX indicates the type of syntax that is unterminated.
320 The job of this function is to move (point) to a new logical location
321 so that analysis can continue, if possible."
323 (funcall semantic-lex-unterminated-syntax-end-function
325 (car semantic-lex-analysis-bounds
)
326 (cdr semantic-lex-analysis-bounds
)
328 (setq semantic-lex-end-point
(point)))
330 ;;; Type table handling.
332 ;; The lexical type table manages types that occur in a grammar file
333 ;; with the %type declaration. Types represent different syntaxes.
334 ;; See code for `semantic-lex-preset-default-types' for the classic
336 (defvar semantic-lex-types-obarray nil
337 "Buffer local types obarray for the lexical analyzer.")
338 (make-variable-buffer-local 'semantic-lex-types-obarray
)
340 (defmacro semantic-lex-type-invalid
(type)
341 "Signal that TYPE is an invalid lexical type name."
342 `(signal 'wrong-type-argument
'(semantic-lex-type-p ,type
)))
344 (defsubst semantic-lex-type-symbol
(type)
345 "Return symbol with TYPE or nil if not found."
346 (and (arrayp semantic-lex-types-obarray
)
348 (intern-soft type semantic-lex-types-obarray
)))
350 (defsubst semantic-lex-type-p
(type)
351 "Return non-nil if a symbol with TYPE name exists."
352 (and (setq type
(semantic-lex-type-symbol type
))
353 (symbol-value type
)))
355 (defsubst semantic-lex-type-set
(type value
)
356 "Set value of symbol with TYPE name to VALUE and return VALUE."
357 (set (intern type semantic-lex-types-obarray
) value
))
359 (defsubst semantic-lex-type-value
(type &optional noerror
)
360 "Return value of symbol with TYPE name.
361 If optional argument NOERROR is non-nil return nil if a symbol with
362 TYPE name does not exist. Otherwise signal an error."
363 (let ((sym (semantic-lex-type-symbol type
)))
367 (semantic-lex-type-invalid type
)))))
369 (defsubst semantic-lex-type-put
(type property value
&optional add
)
370 "For symbol with TYPE name, set its PROPERTY to VALUE.
371 If optional argument ADD is non-nil, create a new symbol with TYPE
372 name if it does not already exist. Otherwise signal an error."
373 (let ((sym (semantic-lex-type-symbol type
)))
375 (or add
(semantic-lex-type-invalid type
))
376 (semantic-lex-type-set type nil
)
377 (setq sym
(semantic-lex-type-symbol type
)))
378 (put sym property value
)))
380 (defsubst semantic-lex-type-get
(type property
&optional noerror
)
381 "For symbol with TYPE name, return its PROPERTY value.
382 If optional argument NOERROR is non-nil return nil if a symbol with
383 TYPE name does not exist. Otherwise signal an error."
384 (let ((sym (semantic-lex-type-symbol type
)))
388 (semantic-lex-type-invalid type
)))))
390 (defun semantic-lex-preset-default-types ()
391 "Install useful default properties for well known types."
392 (semantic-lex-type-put "punctuation" 'matchdatatype
'string t
)
393 (semantic-lex-type-put "punctuation" 'syntax
"\\(\\s.\\|\\s$\\|\\s'\\)+")
394 (semantic-lex-type-put "keyword" 'matchdatatype
'keyword t
)
395 (semantic-lex-type-put "keyword" 'syntax
"\\(\\sw\\|\\s_\\)+")
396 (semantic-lex-type-put "symbol" 'matchdatatype
'regexp t
)
397 (semantic-lex-type-put "symbol" 'syntax
"\\(\\sw\\|\\s_\\)+")
398 (semantic-lex-type-put "string" 'matchdatatype
'sexp t
)
399 (semantic-lex-type-put "string" 'syntax
"\\s\"")
400 (semantic-lex-type-put "number" 'matchdatatype
'regexp t
)
401 (semantic-lex-type-put "number" 'syntax
'semantic-lex-number-expression
)
402 (semantic-lex-type-put "block" 'matchdatatype
'block t
)
403 (semantic-lex-type-put "block" 'syntax
"\\s(\\|\\s)")
406 (defun semantic-lex-make-type-table (specs &optional propspecs
)
407 "Convert type SPECS into an obarray and return it.
408 SPECS must be a list of (TYPE . TOKENS) elements, where:
410 TYPE is the name of the type symbol to define.
411 TOKENS is an list of (TOKSYM . MATCHER) elements, where:
413 TOKSYM is any lexical token symbol.
414 MATCHER is a string or regexp a text must match to be a such
417 If optional argument PROPSPECS is non nil, then interpret it, and
418 apply those properties.
419 PROPSPECS must be a list of (TYPE PROPERTY VALUE)."
420 ;; Create the symbol hash table
421 (let* ((semantic-lex-types-obarray (make-vector 13 0))
422 spec type tokens token alist default
)
423 ;; fill it with stuff
425 (setq spec
(car specs
)
432 (setq token
(car tokens
)
435 (setq alist
(cons token alist
))
436 (setq token
(car token
))
439 "*Warning* default value of <%s> tokens changed to %S, was %S"
441 (setq default token
)))
442 ;; Ensure the default matching spec is the first one.
443 (semantic-lex-type-set type
(cons default
(nreverse alist
))))
444 ;; Install useful default types & properties
445 (semantic-lex-preset-default-types)
446 ;; Apply all properties
448 (setq spec
(car propspecs
)
449 propspecs
(cdr propspecs
))
450 ;; Create the type if necessary.
451 (semantic-lex-type-put (car spec
) (nth 1 spec
) (nth 2 spec
) t
))
452 semantic-lex-types-obarray
))
454 (defsubst semantic-lex-map-types
(fun &optional property
)
455 "Call function FUN on every lexical type.
456 If optional PROPERTY is non-nil, call FUN only on every type symbol
457 which as a PROPERTY value. FUN receives a type symbol as argument."
458 (semantic-lex-map-symbols
459 fun semantic-lex-types-obarray property
))
461 (defun semantic-lex-types (&optional property
)
462 "Return a list of lexical type symbols.
463 If optional PROPERTY is non-nil, return only type symbols which have
466 (semantic-lex-map-types
467 #'(lambda (symbol) (setq types
(cons symbol types
)))
471 ;;; Lexical Analyzer framework settings
474 (defvar semantic-lex-analyzer
'semantic-flex
475 "The lexical analyzer used for a given buffer.
476 See `semantic-lex' for documentation.
477 For compatibility with Semantic 1.x it defaults to `semantic-flex'.")
478 (make-variable-buffer-local 'semantic-lex-analyzer
)
480 (defvar semantic-lex-tokens
494 "An alist of semantic token types.
495 As of December 2001 (semantic 1.4beta13), this variable is not used in
496 any code. The only use is to refer to the doc-string from elsewhere.
498 The key to this alist is the symbol representing token type that
499 \\[semantic-flex] returns. These are
501 - bol: Empty string matching a beginning of line.
502 This token is produced with
503 `semantic-lex-beginning-of-line'.
505 - charquote: String sequences that match `\\s\\+' regexp.
506 This token is produced with `semantic-lex-charquote'.
508 - close-paren: Characters that match `\\s)' regexp.
509 These are typically `)', `}', `]', etc.
510 This token is produced with
511 `semantic-lex-close-paren'.
513 - comment: A comment chunk. These token types are not
515 This token is produced with `semantic-lex-comments'.
516 Comments are ignored with `semantic-lex-ignore-comments'.
517 Comments are treated as whitespace with
518 `semantic-lex-comments-as-whitespace'.
520 - newline Characters matching `\\s-*\\(\n\\|\\s>\\)' regexp.
521 This token is produced with `semantic-lex-newline'.
523 - open-paren: Characters that match `\\s(' regexp.
524 These are typically `(', `{', `[', etc.
525 If `semantic-lex-paren-or-list' is used,
526 then `open-paren' is not usually generated unless
527 the `depth' argument to \\[semantic-lex] is
529 This token is always produced if the analyzer
530 `semantic-lex-open-paren' is used.
532 - punctuation: Characters matching `{\\(\\s.\\|\\s$\\|\\s'\\)'
534 This token is produced with `semantic-lex-punctuation'.
535 Always specify this analyzer after the comment
538 - semantic-list: String delimited by matching parenthesis, braces,
539 etc. that the lexer skipped over, because the
540 `depth' parameter to \\[semantic-flex] was not high
542 This token is produced with `semantic-lex-paren-or-list'.
544 - string: Quoted strings, i.e., string sequences that start
545 and end with characters matching `\\s\"'
546 regexp. The lexer relies on @code{forward-sexp} to
547 find the matching end.
548 This token is produced with `semantic-lex-string'.
550 - symbol: String sequences that match `\\(\\sw\\|\\s_\\)+'
552 This token is produced with
553 `semantic-lex-symbol-or-keyword'. Always add this analyzer
554 after `semantic-lex-number', or other analyzers that
555 match its regular expression.
557 - whitespace: Characters that match `\\s-+' regexp.
558 This token is produced with `semantic-lex-whitespace'.")
560 (defvar semantic-lex-syntax-modifications nil
561 "Changes to the syntax table for this buffer.
562 These changes are active only while the buffer is being flexed.
563 This is a list where each element has the form:
565 CHAR is the char passed to `modify-syntax-entry',
566 and CLASS is the string also passed to `modify-syntax-entry' to define
567 what syntax class CHAR has.")
568 (make-variable-buffer-local 'semantic-lex-syntax-modifications
)
570 (defvar semantic-lex-syntax-table nil
571 "Syntax table used by lexical analysis.
572 See also `semantic-lex-syntax-modifications'.")
573 (make-variable-buffer-local 'semantic-lex-syntax-table
)
575 (defvar semantic-lex-comment-regex nil
576 "Regular expression for identifying comment start during lexical analysis.
577 This may be automatically set when semantic initializes in a mode, but
578 may need to be overridden for some special languages.")
579 (make-variable-buffer-local 'semantic-lex-comment-regex
)
581 (defvar semantic-lex-number-expression
582 ;; This expression was written by David Ponce for Java, and copied
583 ;; here for C and any other similar language.
586 "\\<[0-9]+[.][0-9]+\\([eE][-+]?[0-9]+\\)?[fFdD]?\\>"
588 "\\<[0-9]+[.][eE][-+]?[0-9]+[fFdD]?\\>"
590 "\\<[0-9]+[.][fFdD]\\>"
594 "[.][0-9]+\\([eE][-+]?[0-9]+\\)?[fFdD]?\\>"
596 "\\<[0-9]+[eE][-+]?[0-9]+[fFdD]?\\>"
598 "\\<0[xX][0-9a-fA-F]+[lL]?\\>"
600 "\\<[0-9]+[lLfFdD]?\\>"
603 "Regular expression for matching a number.
604 If this value is nil, no number extraction is done during lex.
605 This expression tries to match C and Java like numbers.
617 <DECIMAL_LITERAL>[lL]?
619 | <OCTAL_LITERAL>[lL]?
624 FLOATING_POINT_LITERAL:
625 [0-9]+[.][0-9]*<EXPONENT>?[fFdD]?
626 | [.][0-9]+<EXPONENT>?[fFdD]?
627 | [0-9]+<EXPONENT>[fFdD]?
628 | [0-9]+<EXPONENT>?[fFdD]
630 (make-variable-buffer-local 'semantic-lex-number-expression
)
632 (defvar semantic-lex-depth
0
633 "Default lexing depth.
634 This specifies how many lists to create tokens in.")
635 (make-variable-buffer-local 'semantic-lex-depth
)
637 (defvar semantic-lex-unterminated-syntax-end-function
638 (lambda (syntax syntax-start lex-end
) lex-end
)
639 "Function called when unterminated syntax is encountered.
640 This should be set to one function. That function should take three
641 parameters. The SYNTAX, or type of syntax which is unterminated.
642 SYNTAX-START where the broken syntax begins.
643 LEX-END is where the lexical analysis was asked to end.
644 This function can be used for languages that can intelligently fix up
645 broken syntax, or the exit lexical analysis via `throw' or `signal'
646 when finding unterminated syntax.")
648 ;;; Interactive testing commands
650 (declare-function semantic-elapsed-time
"semantic")
652 (defun semantic-lex-test (arg)
653 "Test the semantic lexer in the current buffer.
654 If universal argument ARG, then try the whole buffer."
657 (let* ((start (current-time))
658 (result (semantic-lex
659 (if arg
(point-min) (point))
661 (end (current-time)))
662 (message "Elapsed Time: %.2f seconds."
663 (semantic-elapsed-time start end
))
664 (pop-to-buffer "*Lexer Output*")
667 (insert (pp-to-string result
))
668 (goto-char (point-min))
671 (defvar semantic-lex-debug nil
672 "When non-nil, debug the local lexical analyzer.")
674 (defun semantic-lex-debug (arg)
675 "Debug the semantic lexer in the current buffer.
676 Argument ARG specifies of the analyze the whole buffer, or start at point.
677 While engaged, each token identified by the lexer will be highlighted
678 in the target buffer A description of the current token will be
679 displayed in the minibuffer. Press SPC to move to the next lexical token."
681 (require 'semantic
/debug
)
682 (let ((semantic-lex-debug t
))
683 (semantic-lex-test arg
)))
685 (defun semantic-lex-highlight-token (token)
686 "Highlight the lexical TOKEN.
687 TOKEN is a lexical token with a START And END position.
689 (let ((o (semantic-make-overlay (semantic-lex-token-start token
)
690 (semantic-lex-token-end token
))))
691 (semantic-overlay-put o
'face
'highlight
)
694 ;;; Lexical analyzer creation
696 ;; Code for creating a lex function from lists of analyzers.
698 ;; A lexical analyzer is created from a list of individual analyzers.
699 ;; Each individual analyzer specifies a single match, and code that
702 ;; Creation of an analyzer assembles these analyzers into a new function
703 ;; with the behaviors of all the individual analyzers.
705 (defmacro semantic-lex-one-token
(analyzers)
706 "Calculate one token from the current buffer at point.
707 Uses locally bound variables from `define-lex'.
708 Argument ANALYZERS is the list of analyzers being used."
709 (cons 'cond
(mapcar #'symbol-value analyzers
)))
711 (defvar semantic-lex-end-point nil
712 "The end point as tracked through lexical functions.")
714 (defvar semantic-lex-current-depth nil
715 "The current depth as tracked through lexical functions.")
717 (defvar semantic-lex-maximum-depth nil
718 "The maximum depth of parenthesis as tracked through lexical functions.")
720 (defvar semantic-lex-token-stream nil
721 "The current token stream we are collecting.")
723 (defvar semantic-lex-analysis-bounds nil
724 "The bounds of the current analysis.")
726 (defvar semantic-lex-block-streams nil
727 "Streams of tokens inside collapsed blocks.
728 This is an alist of (ANCHOR . STREAM) elements where ANCHOR is the
729 start position of the block, and STREAM is the list of tokens in that
732 (defvar semantic-lex-reset-hooks nil
733 "Abnormal hook used by major-modes to reset lexical analyzers.
734 Hook functions are called with START and END values for the
735 current lexical pass. Should be set with `add-hook', specifying
738 ;; Stack of nested blocks.
739 (defvar semantic-lex-block-stack nil
)
740 ;;(defvar semantic-lex-timeout 5
741 ;; "*Number of sections of lexing before giving up.")
743 (defsubst semantic-lex-debug-break
(token)
744 "Break during lexical analysis at TOKEN."
745 (when semantic-lex-debug
750 (setq o
(semantic-lex-highlight-token token
)))
752 (format "%S :: Depth: %d :: SPC - continue" token semantic-lex-current-depth
))
755 (semantic-overlay-delete o
))))))
757 (defmacro define-lex
(name doc
&rest analyzers
)
758 "Create a new lexical analyzer with NAME.
759 DOC is a documentation string describing this analyzer.
760 ANALYZERS are small code snippets of analyzers to use when
761 building the new NAMED analyzer. Only use analyzers which
762 are written to be used in `define-lex'.
763 Each analyzer should be an analyzer created with `define-lex-analyzer'.
764 Note: The order in which analyzers are listed is important.
765 If two analyzers can match the same text, it is important to order the
766 analyzers so that the one you want to match first occurs first. For
767 example, it is good to put a number analyzer in front of a symbol
768 analyzer which might mistake a number for as a symbol."
769 `(defun ,name
(start end
&optional depth length
)
770 ,(concat doc
"\nSee `semantic-lex' for more information.")
771 ;; Make sure the state of block parsing starts over.
772 (setq semantic-lex-block-streams nil
)
773 ;; Allow specialty reset items.
774 (run-hook-with-args 'semantic-lex-reset-hooks start end
)
776 (let* (;(starttime (current-time))
777 (starting-position (point))
778 (semantic-lex-token-stream nil
)
779 (semantic-lex-block-stack nil
)
781 (semantic-lex-end-point start
)
782 (semantic-lex-current-depth 0)
783 ;; Use the default depth when not specified.
784 (semantic-lex-maximum-depth
785 (or depth semantic-lex-depth
))
786 ;; Bounds needed for unterminated syntax
787 (semantic-lex-analysis-bounds (cons start end
))
788 ;; This entry prevents text properties from
789 ;; confusing our lexical analysis. See Emacs 22 (CVS)
790 ;; version of C++ mode with template hack text properties.
791 (parse-sexp-lookup-properties nil
)
793 ;; Maybe REMOVE THIS LATER.
794 ;; Trying to find incremental parser bug.
795 (when (> end
(point-max))
796 (error ,(format "%s: end (%%d) > point-max (%%d)" name
)
798 (with-syntax-table semantic-lex-syntax-table
800 (while (and (< (point) end
)
802 (<= (length semantic-lex-token-stream
) length
)))
803 (semantic-lex-one-token ,analyzers
)
804 (when (eq semantic-lex-end-point tmp-start
)
805 (error ,(format "%s: endless loop at %%d, after %%S" name
)
806 tmp-start
(car semantic-lex-token-stream
)))
807 (setq tmp-start semantic-lex-end-point
)
808 (goto-char semantic-lex-end-point
)
809 ;;(when (> (semantic-elapsed-time starttime (current-time))
810 ;; semantic-lex-timeout)
811 ;; (error "Timeout during lex at char %d" (point)))
812 (semantic-throw-on-input 'lex
)
813 (semantic-lex-debug-break (car semantic-lex-token-stream
))
815 ;; Check that there is no unterminated block.
816 (when semantic-lex-block-stack
817 (let* ((last (pop semantic-lex-block-stack
))
821 ,(format "%s: `%%s' block from %%S is unterminated" name
)
822 (car blk
) (cadr blk
))
823 (setq blk
(pop semantic-lex-block-stack
)))
824 (semantic-lex-unterminated-syntax-detected (car last
))))
825 ;; Return to where we started.
826 ;; Do not wrap in protective stuff so that if there is an error
827 ;; thrown, the user knows where.
828 (goto-char starting-position
)
829 ;; Return the token stream
830 (nreverse semantic-lex-token-stream
))))
832 ;;; Collapsed block tokens delimited by any tokens.
834 (defun semantic-lex-start-block (syntax)
835 "Mark the last read token as the beginning of a SYNTAX block."
836 (if (or (not semantic-lex-maximum-depth
)
837 (< semantic-lex-current-depth semantic-lex-maximum-depth
))
838 (setq semantic-lex-current-depth
(1+ semantic-lex-current-depth
))
839 (push (list syntax
(car semantic-lex-token-stream
))
840 semantic-lex-block-stack
)))
842 (defun semantic-lex-end-block (syntax)
843 "Process the end of a previously marked SYNTAX block.
844 That is, collapse the tokens inside that block, including the
845 beginning and end of block tokens, into a high level block token of
847 The token at beginning of block is the one marked by a previous call
848 to `semantic-lex-start-block'. The current token is the end of block.
849 The collapsed tokens are saved in `semantic-lex-block-streams'."
850 (if (null semantic-lex-block-stack
)
851 (setq semantic-lex-current-depth
(1- semantic-lex-current-depth
))
852 (let* ((stream semantic-lex-token-stream
)
853 (blk (pop semantic-lex-block-stack
))
855 (first (car bstream
))
856 (last (pop stream
)) ;; The current token mark the EOBLK
858 (if (not (eq (car blk
) syntax
))
859 ;; SYNTAX doesn't match the syntax of the current block in
860 ;; the stack. So we encountered the end of the SYNTAX block
861 ;; before the end of the current one in the stack which is
862 ;; signaled unterminated.
863 (semantic-lex-unterminated-syntax-detected (car blk
))
864 ;; Move tokens found inside the block from the main stream
865 ;; into a separate block stream.
866 (while (and stream
(not (eq (setq tok
(pop stream
)) first
)))
868 ;; The token marked as beginning of block was not encountered.
869 ;; This should not happen!
871 (error "Token %S not found at beginning of block `%s'"
873 ;; Save the block stream for future reuse, to avoid to redo
874 ;; the lexical analysis of the block content!
875 ;; Anchor the block stream with its start position, so we can
876 ;; use: (cdr (assq start semantic-lex-block-streams)) to
877 ;; quickly retrieve the lexical stream associated to a block.
878 (setcar blk
(semantic-lex-token-start first
))
879 (setcdr blk
(nreverse bstream
))
880 (push blk semantic-lex-block-streams
)
881 ;; In the main stream, replace the tokens inside the block by
882 ;; a high level block token of class SYNTAX.
883 (setq semantic-lex-token-stream stream
)
884 (semantic-lex-push-token
886 syntax
(car blk
) (semantic-lex-token-end last
)))
889 ;;; Lexical token API
891 ;; Functions for accessing parts of a token. Use these functions
892 ;; instead of accessing the list structure directly because the
893 ;; contents of the lexical may change.
895 (defmacro semantic-lex-token
(symbol start end
&optional str
)
896 "Create a lexical token.
897 SYMBOL is a symbol representing the class of syntax found.
898 START and END define the bounds of the token in the current buffer.
899 Optional STR is the string for the token only if the bounds in
900 the buffer do not cover the string they represent. (As from
902 ;; This if statement checks the existence of a STR argument at
903 ;; compile time, where STR is some symbol or constant. If the
904 ;; variable STr (runtime) is nil, this will make an incorrect decision.
906 ;; It is like this to maintain the original speed of the compiled
909 `(cons ,symbol
(cons ,str
(cons ,start
,end
)))
910 `(cons ,symbol
(cons ,start
,end
))))
912 (defun semantic-lex-token-p (thing)
913 "Return non-nil if THING is a semantic lex token.
914 This is an exhaustively robust check."
916 (symbolp (car thing
))
917 (or (and (numberp (nth 1 thing
))
918 (numberp (nthcdr 2 thing
)))
919 (and (stringp (nth 1 thing
))
920 (numberp (nth 2 thing
))
921 (numberp (nthcdr 3 thing
)))
925 (defun semantic-lex-token-with-text-p (thing)
926 "Return non-nil if THING is a semantic lex token.
927 This is an exhaustively robust check."
929 (symbolp (car thing
))
931 (stringp (nth 1 thing
))
932 (numberp (nth 2 thing
))
933 (numberp (nth 3 thing
)))
936 (defun semantic-lex-token-without-text-p (thing)
937 "Return non-nil if THING is a semantic lex token.
938 This is an exhaustively robust check."
940 (symbolp (car thing
))
942 (numberp (nth 1 thing
))
943 (numberp (nth 2 thing
)))
948 (defun semantic-lex-expand-block-specs (specs)
949 "Expand block specifications SPECS into a Lisp form.
950 SPECS is a list of (BLOCK BEGIN END) elements where BLOCK, BEGIN, and
951 END are token class symbols that indicate to produce one collapsed
952 BLOCK token from tokens found between BEGIN and END ones.
953 BLOCK must be a non-nil symbol, and at least one of the BEGIN or END
954 symbols must be non-nil too.
955 When BEGIN is non-nil, generate a call to `semantic-lex-start-block'
956 when a BEGIN token class is encountered.
957 When END is non-nil, generate a call to `semantic-lex-end-block' when
958 an END token class is encountered."
959 (let ((class (make-symbol "class"))
964 (push `((eq ',(nth 1 spec
) ,class
)
965 (semantic-lex-start-block ',(car spec
)))
968 (push `((eq ',(nth 2 spec
) ,class
)
969 (semantic-lex-end-block ',(car spec
)))
972 `((let ((,class
(semantic-lex-token-class
973 (car semantic-lex-token-stream
))))
974 (cond ,@(nreverse form
))))
978 (defmacro semantic-lex-push-token
(token &rest blockspecs
)
979 "Push TOKEN in the lexical analyzer token stream.
980 Return the lexical analysis current end point.
981 If optional arguments BLOCKSPECS is non-nil, it specifies to process
982 collapsed block tokens. See `semantic-lex-expand-block-specs' for
984 This macro should only be called within the bounds of
985 `define-lex-analyzer'. It changes the values of the lexical analyzer
986 variables `token-stream' and `semantic-lex-end-point'. If you need to
987 move `semantic-lex-end-point' somewhere else, just modify this
988 variable after calling `semantic-lex-push-token'."
990 (push ,token semantic-lex-token-stream
)
991 ,@(semantic-lex-expand-block-specs blockspecs
)
992 (setq semantic-lex-end-point
993 (semantic-lex-token-end (car semantic-lex-token-stream
)))
996 (defsubst semantic-lex-token-class
(token)
997 "Fetch the class of the lexical token TOKEN.
998 See also the function `semantic-lex-token'."
1001 (defsubst semantic-lex-token-text
(token)
1002 "Fetch the text associated with the lexical token TOKEN.
1003 See also the function `semantic-lex-token'."
1004 (if (stringp (car (cdr token
)))
1006 (buffer-substring-no-properties
1007 (semantic-lex-token-start token
)
1008 (semantic-lex-token-end token
))))
1010 (defun semantic-lex-init ()
1011 "Initialize any lexical state for this buffer."
1012 (unless semantic-lex-comment-regex
1013 (setq semantic-lex-comment-regex
1014 (if comment-start-skip
1015 (concat "\\(\\s<\\|" comment-start-skip
"\\)")
1017 ;; Setup the lexer syntax-table
1018 (setq semantic-lex-syntax-table
(copy-syntax-table (syntax-table)))
1019 (dolist (mod semantic-lex-syntax-modifications
)
1020 (modify-syntax-entry
1021 (car mod
) (nth 1 mod
) semantic-lex-syntax-table
)))
1024 (define-overloadable-function semantic-lex
(start end
&optional depth length
)
1025 "Lexically analyze text in the current buffer between START and END.
1026 Optional argument DEPTH indicates at what level to scan over entire
1027 lists. The last argument, LENGTH specifies that `semantic-lex'
1028 should only return LENGTH tokens. The return value is a token stream.
1029 Each element is a list, such of the form
1030 (symbol start-expression . end-expression)
1031 where SYMBOL denotes the token type.
1032 See `semantic-lex-tokens' variable for details on token types. END
1033 does not mark the end of the text scanned, only the end of the
1034 beginning of text scanned. Thus, if a string extends past END, the
1035 end of the return token will be larger than END. To truly restrict
1036 scanning, use `narrow-to-region'."
1037 (funcall semantic-lex-analyzer start end depth length
))
1039 (defsubst semantic-lex-buffer
(&optional depth
)
1040 "Lex the current buffer.
1041 Optional argument DEPTH is the depth to scan into lists."
1042 (semantic-lex (point-min) (point-max) depth
))
1044 (defsubst semantic-lex-list
(semlist depth
)
1045 "Lex the body of SEMLIST to DEPTH."
1046 (semantic-lex (semantic-lex-token-start semlist
)
1047 (semantic-lex-token-end semlist
)
1050 ;;; Analyzer creation macros
1052 ;; An individual analyzer is a condition and code that goes with it.
1054 ;; Created analyzers become variables with the code associated with them
1055 ;; as the symbol value. These analyzers are assembled into a lexer
1056 ;; to create new lexical analyzers.
1058 (defcustom semantic-lex-debug-analyzers nil
1059 "Non nil means to debug analyzers with syntax protection.
1060 Only in effect if `debug-on-error' is also non-nil."
1064 (defmacro semantic-lex-unterminated-syntax-protection
(syntax &rest forms
)
1065 "For SYNTAX, execute FORMS with protection for unterminated syntax.
1066 If FORMS throws an error, treat this as a syntax problem, and
1067 execute the unterminated syntax code. FORMS should return a position.
1068 Irregardless of an error, the cursor should be moved to the end of
1069 the desired syntax, and a position returned.
1070 If `debug-on-error' is set, errors are not caught, so that you can
1072 Avoid using a large FORMS since it is duplicated."
1073 `(if (and debug-on-error semantic-lex-debug-analyzers
)
1078 (semantic-lex-unterminated-syntax-detected ,syntax
)))))
1079 (put 'semantic-lex-unterminated-syntax-protection
1080 'lisp-indent-function
1)
1082 (defmacro define-lex-analyzer
(name doc condition
&rest forms
)
1083 "Create a single lexical analyzer NAME with DOC.
1084 When an analyzer is called, the current buffer and point are
1085 positioned in a buffer at the location to be analyzed.
1086 CONDITION is an expression which returns t if FORMS should be run.
1087 Within the bounds of CONDITION and FORMS, the use of backquote
1088 can be used to evaluate expressions at compile time.
1089 While forms are running, the following variables will be locally bound:
1090 `semantic-lex-analysis-bounds' - The bounds of the current analysis.
1091 of the form (START . END)
1092 `semantic-lex-maximum-depth' - The maximum depth of semantic-list
1093 for the current analysis.
1094 `semantic-lex-current-depth' - The current depth of `semantic-list' that has
1096 `semantic-lex-end-point' - End Point after match.
1097 Analyzers should set this to a buffer location if their
1098 match string does not represent the end of the matched text.
1099 `semantic-lex-token-stream' - The token list being collected.
1100 Add new lexical tokens to this list.
1101 Proper action in FORMS is to move the value of `semantic-lex-end-point' to
1102 after the location of the analyzed entry, and to add any discovered tokens
1103 at the beginning of `semantic-lex-token-stream'.
1104 This can be done by using `semantic-lex-push-token'."
1106 (defvar ,name nil
,doc
)
1108 ;; Do this part separately so that re-evaluation rebuilds this code.
1109 (setq ,name
'(,condition
,@forms
))
1110 ;; Build a single lexical analyzer function, so the doc for
1111 ;; function help is automatically provided, and perhaps the
1112 ;; function could be useful for testing and debugging one
1114 (fset ',name
(lambda () ,doc
1115 (let ((semantic-lex-token-stream nil
)
1116 (semantic-lex-end-point (point))
1117 (semantic-lex-analysis-bounds
1118 (cons (point) (point-max)))
1119 (semantic-lex-current-depth 0)
1120 (semantic-lex-maximum-depth
1123 (when ,condition
,@forms
)
1124 semantic-lex-token-stream
)))
1127 (defmacro define-lex-regex-analyzer
(name doc regexp
&rest forms
)
1128 "Create a lexical analyzer with NAME and DOC that will match REGEXP.
1129 FORMS are evaluated upon a successful match.
1130 See `define-lex-analyzer' for more about analyzers."
1131 `(define-lex-analyzer ,name
1133 (looking-at ,regexp
)
1137 (defmacro define-lex-simple-regex-analyzer
(name doc regexp toksym
1140 "Create a lexical analyzer with NAME and DOC that match REGEXP.
1141 TOKSYM is the symbol to use when creating a semantic lexical token.
1142 INDEX is the index into the match that defines the bounds of the token.
1143 Index should be a plain integer, and not specified in the macro as an
1145 FORMS are evaluated upon a successful match BEFORE the new token is
1146 created. It is valid to ignore FORMS.
1147 See `define-lex-analyzer' for more about analyzers."
1148 `(define-lex-analyzer ,name
1150 (looking-at ,regexp
)
1152 (semantic-lex-push-token
1153 (semantic-lex-token ,toksym
1154 (match-beginning ,(or index
0))
1155 (match-end ,(or index
0))))
1158 (defmacro define-lex-block-analyzer
(name doc spec1
&rest specs
)
1159 "Create a lexical analyzer NAME for paired delimiters blocks.
1160 It detects a paired delimiters block or the corresponding open or
1161 close delimiter depending on the value of the variable
1162 `semantic-lex-current-depth'. DOC is the documentation string of the lexical
1163 analyzer. SPEC1 and SPECS specify the token symbols and open, close
1164 delimiters used. Each SPEC has the form:
1166 \(BLOCK-SYM (OPEN-DELIM OPEN-SYM) (CLOSE-DELIM CLOSE-SYM))
1168 where BLOCK-SYM is the symbol returned in a block token. OPEN-DELIM
1169 and CLOSE-DELIM are respectively the open and close delimiters
1170 identifying a block. OPEN-SYM and CLOSE-SYM are respectively the
1171 symbols returned in open and close tokens."
1172 (let ((specs (cons spec1 specs
))
1173 spec open olist clist
)
1175 (setq spec
(car specs
)
1178 ;; build alist ((OPEN-DELIM OPEN-SYM BLOCK-SYM) ...)
1179 olist
(cons (list (car open
) (cadr open
) (car spec
)) olist
)
1180 ;; build alist ((CLOSE-DELIM CLOSE-SYM) ...)
1181 clist
(cons (nth 2 spec
) clist
)))
1182 `(define-lex-analyzer ,name
1185 (looking-at "\\(\\s(\\|\\s)\\)")
1186 (let ((text (match-string 0)) match
)
1188 ((setq match
(assoc text
',olist
))
1189 (if (or (not semantic-lex-maximum-depth
)
1190 (< semantic-lex-current-depth semantic-lex-maximum-depth
))
1192 (setq semantic-lex-current-depth
(1+ semantic-lex-current-depth
))
1193 (semantic-lex-push-token
1196 (match-beginning 0) (match-end 0))))
1197 (semantic-lex-push-token
1202 (semantic-lex-unterminated-syntax-protection (nth 2 match
)
1207 ((setq match
(assoc text
',clist
))
1208 (if (> semantic-lex-current-depth
0)
1210 (setq semantic-lex-current-depth
(1- semantic-lex-current-depth
))
1211 (semantic-lex-push-token
1214 (match-beginning 0) (match-end 0)))))))))
1219 ;; Pre-defined common analyzers.
1221 (define-lex-analyzer semantic-lex-default-action
1222 "The default action when no other lexical actions match text.
1223 This action will just throw an error."
1225 (error "Unmatched Text during Lexical Analysis"))
1227 (define-lex-analyzer semantic-lex-beginning-of-line
1228 "Detect and create a beginning of line token (BOL)."
1230 ;; Just insert a (bol N . N) token in the token stream,
1231 ;; without moving the point. N is the point at the
1232 ;; beginning of line.
1233 (semantic-lex-push-token (semantic-lex-token 'bol
(point) (point)))
1235 ;; We identify and add the BOL token onto the stream, but since
1236 ;; semantic-lex-end-point doesn't move, we always fail CONDITION, and have no
1240 (define-lex-simple-regex-analyzer semantic-lex-newline
1241 "Detect and create newline tokens."
1242 "\\s-*\\(\n\\|\\s>\\)" 'newline
1)
1244 (define-lex-regex-analyzer semantic-lex-newline-as-whitespace
1245 "Detect and create newline tokens.
1246 Use this ONLY if newlines are not whitespace characters (such as when
1247 they are comment end characters) AND when you want whitespace tokens."
1248 "\\s-*\\(\n\\|\\s>\\)"
1249 ;; Language wants whitespaces. Create a token for it.
1250 (if (eq (semantic-lex-token-class (car semantic-lex-token-stream
))
1252 ;; Merge whitespace tokens together if they are adjacent. Two
1253 ;; whitespace tokens may be separated by a comment which is not in
1254 ;; the token stream.
1255 (setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream
))
1257 (semantic-lex-push-token
1259 'whitespace
(match-beginning 0) (match-end 0)))))
1261 (define-lex-regex-analyzer semantic-lex-ignore-newline
1262 "Detect and ignore newline tokens.
1263 Use this ONLY if newlines are not whitespace characters (such as when
1264 they are comment end characters)."
1265 "\\s-*\\(\n\\|\\s>\\)"
1266 (setq semantic-lex-end-point
(match-end 0)))
1268 (define-lex-regex-analyzer semantic-lex-whitespace
1269 "Detect and create whitespace tokens."
1270 ;; catch whitespace when needed
1272 ;; Language wants whitespaces. Create a token for it.
1273 (if (eq (semantic-lex-token-class (car semantic-lex-token-stream
))
1275 ;; Merge whitespace tokens together if they are adjacent. Two
1276 ;; whitespace tokens may be separated by a comment which is not in
1277 ;; the token stream.
1279 (setq semantic-lex-end-point
(match-end 0))
1280 (setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream
))
1281 semantic-lex-end-point
))
1282 (semantic-lex-push-token
1284 'whitespace
(match-beginning 0) (match-end 0)))))
1286 (define-lex-regex-analyzer semantic-lex-ignore-whitespace
1287 "Detect and skip over whitespace tokens."
1288 ;; catch whitespace when needed
1290 ;; Skip over the detected whitespace, do not create a token for it.
1291 (setq semantic-lex-end-point
(match-end 0)))
1293 (define-lex-simple-regex-analyzer semantic-lex-number
1294 "Detect and create number tokens.
1295 See `semantic-lex-number-expression' for details on matching numbers,
1296 and number formats."
1297 semantic-lex-number-expression
'number
)
1299 (define-lex-regex-analyzer semantic-lex-symbol-or-keyword
1300 "Detect and create symbol and keyword tokens."
1301 "\\(\\sw\\|\\s_\\)+"
1302 (semantic-lex-push-token
1304 (or (semantic-lex-keyword-p (match-string 0)) 'symbol
)
1305 (match-beginning 0) (match-end 0))))
1307 (define-lex-simple-regex-analyzer semantic-lex-charquote
1308 "Detect and create charquote tokens."
1309 ;; Character quoting characters (ie, \n as newline)
1310 "\\s\\+" 'charquote
)
1312 (define-lex-simple-regex-analyzer semantic-lex-punctuation
1313 "Detect and create punctuation tokens."
1314 "\\(\\s.\\|\\s$\\|\\s'\\)" 'punctuation
)
1316 (define-lex-analyzer semantic-lex-punctuation-type
1317 "Detect and create a punctuation type token.
1318 Recognized punctuation is defined in the current table of lexical
1319 types, as the value of the `punctuation' token type."
1320 (and (looking-at "\\(\\s.\\|\\s$\\|\\s'\\)+")
1321 (let* ((key (match-string 0))
1322 (pos (match-beginning 0))
1325 (lst (semantic-lex-type-value "punctuation" t
))
1326 (def (car lst
)) ;; default lexical symbol or nil
1327 (lst (cdr lst
)) ;; alist of (LEX-SYM . PUNCT-STRING)
1330 ;; Starting with the longest one, search if the
1331 ;; punctuation string is defined for this language.
1332 (while (and (> len
0) (not (setq elt
(rassoc key lst
))))
1334 key
(substring key
0 len
))))
1335 (if elt
;; Return the punctuation token found
1336 (semantic-lex-push-token
1337 (semantic-lex-token (car elt
) pos
(+ pos len
)))
1338 (if def
;; Return a default generic token
1339 (semantic-lex-push-token
1340 (semantic-lex-token def pos end
))
1344 (define-lex-regex-analyzer semantic-lex-paren-or-list
1345 "Detect open parenthesis.
1346 Return either a paren token or a semantic list token depending on
1347 `semantic-lex-current-depth'."
1349 (if (or (not semantic-lex-maximum-depth
)
1350 (< semantic-lex-current-depth semantic-lex-maximum-depth
))
1352 (setq semantic-lex-current-depth
(1+ semantic-lex-current-depth
))
1353 (semantic-lex-push-token
1355 'open-paren
(match-beginning 0) (match-end 0))))
1356 (semantic-lex-push-token
1358 'semantic-list
(match-beginning 0)
1360 (semantic-lex-unterminated-syntax-protection 'semantic-list
1366 (define-lex-simple-regex-analyzer semantic-lex-open-paren
1367 "Detect and create an open parenthesis token."
1368 "\\s(" 'open-paren
0 (setq semantic-lex-current-depth
(1+ semantic-lex-current-depth
)))
1370 (define-lex-simple-regex-analyzer semantic-lex-close-paren
1371 "Detect and create a close parenthesis token."
1372 "\\s)" 'close-paren
0 (setq semantic-lex-current-depth
(1- semantic-lex-current-depth
)))
1374 (define-lex-regex-analyzer semantic-lex-string
1375 "Detect and create a string token."
1377 ;; Zing to the end of this string.
1378 (semantic-lex-push-token
1382 (semantic-lex-unterminated-syntax-protection 'string
1387 (define-lex-regex-analyzer semantic-lex-comments
1388 "Detect and create a comment token."
1389 semantic-lex-comment-regex
1392 ;; Generate newline token if enabled
1393 (if (bolp) (backward-char 1))
1394 (setq semantic-lex-end-point
(point))
1395 ;; Language wants comments or want them as whitespaces,
1396 ;; link them together.
1397 (if (eq (semantic-lex-token-class (car semantic-lex-token-stream
)) 'comment
)
1398 (setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream
))
1399 semantic-lex-end-point
)
1400 (semantic-lex-push-token
1402 'comment
(match-beginning 0) semantic-lex-end-point
)))))
1404 (define-lex-regex-analyzer semantic-lex-comments-as-whitespace
1405 "Detect comments and create a whitespace token."
1406 semantic-lex-comment-regex
1409 ;; Generate newline token if enabled
1410 (if (bolp) (backward-char 1))
1411 (setq semantic-lex-end-point
(point))
1412 ;; Language wants comments or want them as whitespaces,
1413 ;; link them together.
1414 (if (eq (semantic-lex-token-class (car semantic-lex-token-stream
)) 'whitespace
)
1415 (setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream
))
1416 semantic-lex-end-point
)
1417 (semantic-lex-push-token
1419 'whitespace
(match-beginning 0) semantic-lex-end-point
)))))
1421 (define-lex-regex-analyzer semantic-lex-ignore-comments
1422 "Detect and create a comment token."
1423 semantic-lex-comment-regex
1424 (let ((comment-start-point (point)))
1426 (if (eq (point) comment-start-point
)
1427 ;; In this case our start-skip string failed
1428 ;; to work properly. Lets try and move over
1429 ;; whatever white space we matched to begin
1431 (skip-syntax-forward "-.'" (point-at-eol))
1432 ;; We may need to back up so newlines or whitespace is generated.
1435 (if (eq (point) comment-start-point
)
1436 (error "Strange comment syntax prevents lexical analysis"))
1437 (setq semantic-lex-end-point
(point))))
1441 ;; Predefined lexers that could be used instead of creating new
1444 (define-lex semantic-comment-lexer
1445 "A simple lexical analyzer that handles comments.
1446 This lexer will only return comment tokens. It is the default lexer
1447 used by `semantic-find-doc-snarf-comment' to snarf up the comment at
1449 semantic-lex-ignore-whitespace
1450 semantic-lex-ignore-newline
1451 semantic-lex-comments
1452 semantic-lex-default-action
)
1456 (define-lex semantic-simple-lexer
1457 "A simple lexical analyzer that handles simple buffers.
1458 This lexer ignores comments and whitespace, and will return
1459 syntax as specified by the syntax table."
1460 semantic-lex-ignore-whitespace
1461 semantic-lex-ignore-newline
1463 semantic-lex-symbol-or-keyword
1464 semantic-lex-charquote
1465 semantic-lex-paren-or-list
1466 semantic-lex-close-paren
1468 semantic-lex-ignore-comments
1469 semantic-lex-punctuation
1470 semantic-lex-default-action
)
1472 ;;; Analyzers generated from grammar.
1474 ;; Some analyzers are hand written. Analyzers created with these
1475 ;; functions are generated from the grammar files.
1477 (defmacro define-lex-keyword-type-analyzer
(name doc syntax
)
1478 "Define a keyword type analyzer NAME with DOC string.
1479 SYNTAX is the regexp that matches a keyword syntactic expression."
1480 (let ((key (make-symbol "key")))
1481 `(define-lex-analyzer ,name
1483 (and (looking-at ,syntax
)
1484 (let ((,key
(semantic-lex-keyword-p (match-string 0))))
1486 (semantic-lex-push-token
1488 ,key
(match-beginning 0) (match-end 0)))))))
1491 (defmacro define-lex-sexp-type-analyzer
(name doc syntax token
)
1492 "Define a sexp type analyzer NAME with DOC string.
1493 SYNTAX is the regexp that matches the beginning of the s-expression.
1494 TOKEN is the lexical token returned when SYNTAX matches."
1495 `(define-lex-regex-analyzer ,name
1498 (semantic-lex-push-token
1502 (semantic-lex-unterminated-syntax-protection ,token
1507 (defmacro define-lex-regex-type-analyzer
(name doc syntax matches default
)
1508 "Define a regexp type analyzer NAME with DOC string.
1509 SYNTAX is the regexp that matches a syntactic expression.
1510 MATCHES is an alist of lexical elements used to refine the syntactic
1512 DEFAULT is the default lexical token returned when no MATCHES."
1514 (let* ((val (make-symbol "val"))
1515 (lst (make-symbol "lst"))
1516 (elt (make-symbol "elt"))
1517 (pos (make-symbol "pos"))
1518 (end (make-symbol "end")))
1519 `(define-lex-analyzer ,name
1521 (and (looking-at ,syntax
)
1522 (let* ((,val
(match-string 0))
1523 (,pos
(match-beginning 0))
1524 (,end
(match-end 0))
1527 (while (and ,lst
(not ,elt
))
1528 (if (string-match (cdar ,lst
) ,val
)
1529 (setq ,elt
(caar ,lst
))
1530 (setq ,lst
(cdr ,lst
))))
1531 (semantic-lex-push-token
1532 (semantic-lex-token (or ,elt
,default
) ,pos
,end
))))
1534 `(define-lex-simple-regex-analyzer ,name
1539 (defmacro define-lex-string-type-analyzer
(name doc syntax matches default
)
1540 "Define a string type analyzer NAME with DOC string.
1541 SYNTAX is the regexp that matches a syntactic expression.
1542 MATCHES is an alist of lexical elements used to refine the syntactic
1544 DEFAULT is the default lexical token returned when no MATCHES."
1546 (let* ((val (make-symbol "val"))
1547 (lst (make-symbol "lst"))
1548 (elt (make-symbol "elt"))
1549 (pos (make-symbol "pos"))
1550 (end (make-symbol "end"))
1551 (len (make-symbol "len")))
1552 `(define-lex-analyzer ,name
1554 (and (looking-at ,syntax
)
1555 (let* ((,val
(match-string 0))
1556 (,pos
(match-beginning 0))
1557 (,end
(match-end 0))
1558 (,len
(- ,end
,pos
))
1561 ;; Starting with the longest one, search if a lexical
1562 ;; value match a token defined for this language.
1563 (while (and (> ,len
0) (not (setq ,elt
(rassoc ,val
,lst
))))
1564 (setq ,len
(1- ,len
)
1565 ,val
(substring ,val
0 ,len
)))
1566 (when ,elt
;; Adjust token end position.
1567 (setq ,elt
(car ,elt
)
1568 ,end
(+ ,pos
,len
)))
1569 (semantic-lex-push-token
1570 (semantic-lex-token (or ,elt
,default
) ,pos
,end
))))
1572 `(define-lex-simple-regex-analyzer ,name
1577 (defmacro define-lex-block-type-analyzer
(name doc syntax matches
)
1578 "Define a block type analyzer NAME with DOC string.
1580 SYNTAX is the regexp that matches block delimiters, typically the
1581 open (`\\\\s(') and close (`\\\\s)') parenthesis syntax classes.
1583 MATCHES is a pair (OPEN-SPECS . CLOSE-SPECS) that defines blocks.
1585 OPEN-SPECS is a list of (OPEN-DELIM OPEN-TOKEN BLOCK-TOKEN) elements
1588 OPEN-DELIM is a string: the block open delimiter character.
1590 OPEN-TOKEN is the lexical token class associated to the OPEN-DELIM
1593 BLOCK-TOKEN is the lexical token class associated to the block
1594 that starts at the OPEN-DELIM delimiter.
1596 CLOSE-SPECS is a list of (CLOSE-DELIM CLOSE-TOKEN) elements where:
1598 CLOSE-DELIM is a string: the block end delimiter character.
1600 CLOSE-TOKEN is the lexical token class associated to the
1601 CLOSE-DELIM delimiter.
1603 Each element in OPEN-SPECS must have a corresponding element in
1606 The lexer will return a BLOCK-TOKEN token when the value of
1607 `semantic-lex-current-depth' is greater than or equal to the maximum
1608 depth of parenthesis tracking (see also the function `semantic-lex').
1609 Otherwise it will return OPEN-TOKEN and CLOSE-TOKEN tokens.
1611 TO DO: Put the following in the developer's guide and just put a
1616 The value of a block token must be a string that contains a readable
1619 \"(OPEN-TOKEN CLOSE-TOKEN)\"
1621 OPEN-TOKEN and CLOSE-TOKEN represent the block delimiters, and must be
1622 lexical tokens of respectively `open-paren' and `close-paren' types.
1623 Their value is the corresponding delimiter character as a string.
1625 Here is a small example to analyze a parenthesis block:
1627 %token <block> PAREN_BLOCK \"(LPAREN RPAREN)\"
1628 %token <open-paren> LPAREN \"(\"
1629 %token <close-paren> RPAREN \")\"
1631 When the lexer encounters the open-paren delimiter \"(\":
1633 - If the maximum depth of parenthesis tracking is not reached (that
1634 is, current depth < max depth), it returns a (LPAREN start . end)
1635 token, then continue analysis inside the block. Later, when the
1636 corresponding close-paren delimiter \")\" will be encountered, it
1637 will return a (RPAREN start . end) token.
1639 - If the maximum depth of parenthesis tracking is reached (current
1640 depth >= max depth), it returns the whole parenthesis block as
1641 a (PAREN_BLOCK start . end) token."
1642 (let* ((val (make-symbol "val"))
1643 (lst (make-symbol "lst"))
1644 (elt (make-symbol "elt")))
1645 `(define-lex-analyzer ,name
1648 (looking-at ,syntax
) ;; "\\(\\s(\\|\\s)\\)"
1649 (let ((,val
(match-string 0))
1653 ((setq ,elt
(assoc ,val
(car ,lst
)))
1654 (if (or (not semantic-lex-maximum-depth
)
1655 (< semantic-lex-current-depth semantic-lex-maximum-depth
))
1657 (setq semantic-lex-current-depth
(1+ semantic-lex-current-depth
))
1658 (semantic-lex-push-token
1661 (match-beginning 0) (match-end 0))))
1662 (semantic-lex-push-token
1667 (semantic-lex-unterminated-syntax-protection (nth 2 ,elt
)
1670 ((setq ,elt
(assoc ,val
(cdr ,lst
)))
1671 (setq semantic-lex-current-depth
(1- semantic-lex-current-depth
))
1672 (semantic-lex-push-token
1675 (match-beginning 0) (match-end 0))))
1681 ;; The semantic lexers, unlike other lexers, can throw errors on
1682 ;; unbalanced syntax. Since editing is all about changing text
1683 ;; we need to provide a convenient way to protect against syntactic
1686 (defmacro semantic-lex-catch-errors
(symbol &rest forms
)
1687 "Using SYMBOL, execute FORMS catching lexical errors.
1688 If FORMS results in a call to the parser that throws a lexical error,
1689 the error will be caught here without the buffer's cache being thrown
1691 If there is an error, the syntax that failed is returned.
1692 If there is no error, then the last value of FORMS is returned."
1693 (let ((ret (make-symbol "ret"))
1694 (syntax (make-symbol "syntax"))
1695 (start (make-symbol "start"))
1696 (end (make-symbol "end")))
1697 `(let* ((semantic-lex-unterminated-syntax-end-function
1698 (lambda (,syntax
,start
,end
)
1699 (throw ',symbol
,syntax
)))
1700 ;; Delete the below when semantic-flex is fully retired.
1701 (semantic-flex-unterminated-syntax-end-function
1702 semantic-lex-unterminated-syntax-end-function
)
1703 (,ret
(catch ',symbol
1707 ;; Great Sadness. Assume that FORMS execute within the
1708 ;; confines of the current buffer only! Mark this thing
1709 ;; unparsable iff the special symbol was thrown. This
1710 ;; will prevent future calls from parsing, but will allow
1711 ;; then to still return the cache.
1713 ;; Leave this message off. If an APP using this fcn wants
1714 ;; a message, they can do it themselves. This cleans up
1715 ;; problems with the idle scheduler obscuring useful data.
1716 ;;(message "Buffer not currently parsable (%S)." ,ret)
1717 (semantic-parse-tree-unparseable))
1719 (put 'semantic-lex-catch-errors
'lisp-indent-function
1)
1722 ;;; Interfacing with edebug
1728 (def-edebug-spec define-lex
1729 (&define name stringp
(&rest symbolp
))
1731 (def-edebug-spec define-lex-analyzer
1732 (&define name stringp form def-body
)
1734 (def-edebug-spec define-lex-regex-analyzer
1735 (&define name stringp form def-body
)
1737 (def-edebug-spec define-lex-simple-regex-analyzer
1738 (&define name stringp form symbolp
[ &optional form
] def-body
)
1740 (def-edebug-spec define-lex-block-analyzer
1741 (&define name stringp form
(&rest form
))
1743 (def-edebug-spec semantic-lex-catch-errors
1749 ;;; Compatibility with Semantic 1.x lexical analysis
1751 ;; NOTE: DELETE THIS SOMEDAY SOON
1753 (semantic-alias-obsolete 'semantic-flex-start
'semantic-lex-token-start
"23.2")
1754 (semantic-alias-obsolete 'semantic-flex-end
'semantic-lex-token-end
"23.2")
1755 (semantic-alias-obsolete 'semantic-flex-text
'semantic-lex-token-text
"23.2")
1756 (semantic-alias-obsolete 'semantic-flex-make-keyword-table
'semantic-lex-make-keyword-table
"23.2")
1757 (semantic-alias-obsolete 'semantic-flex-keyword-p
'semantic-lex-keyword-p
"23.2")
1758 (semantic-alias-obsolete 'semantic-flex-keyword-put
'semantic-lex-keyword-put
"23.2")
1759 (semantic-alias-obsolete 'semantic-flex-keyword-get
'semantic-lex-keyword-get
"23.2")
1760 (semantic-alias-obsolete 'semantic-flex-map-keywords
'semantic-lex-map-keywords
"23.2")
1761 (semantic-alias-obsolete 'semantic-flex-keywords
'semantic-lex-keywords
"23.2")
1762 (semantic-alias-obsolete 'semantic-flex-buffer
'semantic-lex-buffer
"23.2")
1763 (semantic-alias-obsolete 'semantic-flex-list
'semantic-lex-list
"23.2")
1765 ;; This simple scanner uses the syntax table to generate a stream of
1766 ;; simple tokens of the form:
1768 ;; (SYMBOL START . END)
1770 ;; Where symbol is the type of thing it is. START and END mark that
1771 ;; objects boundary.
1773 (defvar semantic-flex-tokens semantic-lex-tokens
1774 "An alist of semantic token types.
1775 See variable `semantic-lex-tokens'.")
1777 (defvar semantic-flex-unterminated-syntax-end-function
1778 (lambda (syntax syntax-start flex-end
) flex-end
)
1779 "Function called when unterminated syntax is encountered.
1780 This should be set to one function. That function should take three
1781 parameters. The SYNTAX, or type of syntax which is unterminated.
1782 SYNTAX-START where the broken syntax begins.
1783 FLEX-END is where the lexical analysis was asked to end.
1784 This function can be used for languages that can intelligently fix up
1785 broken syntax, or the exit lexical analysis via `throw' or `signal'
1786 when finding unterminated syntax.")
1788 (defvar semantic-flex-extensions nil
1789 "Buffer local extensions to the lexical analyzer.
1790 This should contain an alist with a key of a regex and a data element of
1791 a function. The function should both move point, and return a lexical
1794 nil is also a valid return value.
1795 TYPE can be any type of symbol, as long as it doesn't occur as a
1796 nonterminal in the language definition.")
1797 (make-variable-buffer-local 'semantic-flex-extensions
)
1799 (defvar semantic-flex-syntax-modifications nil
1800 "Changes to the syntax table for this buffer.
1801 These changes are active only while the buffer is being flexed.
1802 This is a list where each element has the form:
1804 CHAR is the char passed to `modify-syntax-entry',
1805 and CLASS is the string also passed to `modify-syntax-entry' to define
1806 what syntax class CHAR has.")
1807 (make-variable-buffer-local 'semantic-flex-syntax-modifications
)
1809 (defvar semantic-ignore-comments t
1810 "Default comment handling.
1811 The value t means to strip comments when flexing; nil means
1812 to keep comments as part of the token stream.")
1813 (make-variable-buffer-local 'semantic-ignore-comments
)
1815 (defvar semantic-flex-enable-newlines nil
1816 "When flexing, report 'newlines as syntactic elements.
1817 Useful for languages where the newline is a special case terminator.
1818 Only set this on a per mode basis, not globally.")
1819 (make-variable-buffer-local 'semantic-flex-enable-newlines
)
1821 (defvar semantic-flex-enable-whitespace nil
1822 "When flexing, report 'whitespace as syntactic elements.
1823 Useful for languages where the syntax is whitespace dependent.
1824 Only set this on a per mode basis, not globally.")
1825 (make-variable-buffer-local 'semantic-flex-enable-whitespace
)
1827 (defvar semantic-flex-enable-bol nil
1828 "When flexing, report beginning of lines as syntactic elements.
1829 Useful for languages like python which are indentation sensitive.
1830 Only set this on a per mode basis, not globally.")
1831 (make-variable-buffer-local 'semantic-flex-enable-bol
)
1833 (defvar semantic-number-expression semantic-lex-number-expression
1834 "See variable `semantic-lex-number-expression'.")
1835 (make-variable-buffer-local 'semantic-number-expression
)
1837 (defvar semantic-flex-depth
0
1838 "Default flexing depth.
1839 This specifies how many lists to create tokens in.")
1840 (make-variable-buffer-local 'semantic-flex-depth
)
1842 (defun semantic-flex (start end
&optional depth length
)
1843 "Using the syntax table, do something roughly equivalent to flex.
1844 Semantically check between START and END. Optional argument DEPTH
1845 indicates at what level to scan over entire lists.
1846 The return value is a token stream. Each element is a list, such of
1847 the form (symbol start-expression . end-expression) where SYMBOL
1848 denotes the token type.
1849 See `semantic-flex-tokens' variable for details on token types.
1850 END does not mark the end of the text scanned, only the end of the
1851 beginning of text scanned. Thus, if a string extends past END, the
1852 end of the return token will be larger than END. To truly restrict
1853 scanning, use `narrow-to-region'.
1854 The last argument, LENGTH specifies that `semantic-flex' should only
1855 return LENGTH tokens."
1856 (message "`semantic-flex' is an obsolete function. Use `define-lex' to create lexers.")
1857 (if (not semantic-flex-keywords-obarray
)
1858 (setq semantic-flex-keywords-obarray
[ nil
]))
1863 (cs (if comment-start-skip
1864 (concat "\\(\\s<\\|" comment-start-skip
"\\)")
1865 (concat "\\(\\s<\\)")))
1866 (newsyntax (copy-syntax-table (syntax-table)))
1867 (mods semantic-flex-syntax-modifications
)
1868 ;; Use the default depth if it is not specified.
1869 (depth (or depth semantic-flex-depth
)))
1870 ;; Update the syntax table
1872 (modify-syntax-entry (car (car mods
)) (car (cdr (car mods
))) newsyntax
)
1873 (setq mods
(cdr mods
)))
1874 (with-syntax-table newsyntax
1876 (while (and (< (point) end
) (or (not length
) (<= (length ts
) length
)))
1878 ;; catch beginning of lines when needed.
1879 ;; Must be done before catching any other tokens!
1880 ((and semantic-flex-enable-bol
1882 ;; Just insert a (bol N . N) token in the token stream,
1883 ;; without moving the point. N is the point at the
1884 ;; beginning of line.
1885 (setq ts
(cons (cons 'bol
(cons (point) (point))) ts
))
1887 ;; special extensions, includes whitespace, nl, etc.
1888 ((and semantic-flex-extensions
1889 (let ((fe semantic-flex-extensions
)
1892 (if (looking-at (car (car fe
)))
1893 (setq ts
(cons (funcall (cdr (car fe
))) ts
)
1898 (if (and r
(not (car ts
))) (setq ts
(cdr ts
)))
1900 ;; catch newlines when needed
1901 ((looking-at "\\s-*\\(\n\\|\\s>\\)")
1902 (if semantic-flex-enable-newlines
1903 (setq ep
(match-end 1)
1904 ts
(cons (cons 'newline
1905 (cons (match-beginning 1) ep
))
1907 ;; catch whitespace when needed
1908 ((looking-at "\\s-+")
1909 (if semantic-flex-enable-whitespace
1910 ;; Language wants whitespaces, link them together.
1911 (if (eq (car (car ts
)) 'whitespace
)
1912 (setcdr (cdr (car ts
)) (match-end 0))
1913 (setq ts
(cons (cons 'whitespace
1914 (cons (match-beginning 0)
1918 ((and semantic-number-expression
1919 (looking-at semantic-number-expression
))
1920 (setq ts
(cons (cons 'number
1921 (cons (match-beginning 0)
1925 ((looking-at "\\(\\sw\\|\\s_\\)+")
1926 (setq ts
(cons (cons
1927 ;; Get info on if this is a keyword or not
1928 (or (semantic-lex-keyword-p (match-string 0))
1930 (cons (match-beginning 0) (match-end 0)))
1932 ;; Character quoting characters (ie, \n as newline)
1933 ((looking-at "\\s\\+")
1934 (setq ts
(cons (cons 'charquote
1935 (cons (match-beginning 0) (match-end 0)))
1937 ;; Open parens, or semantic-lists.
1938 ((looking-at "\\s(")
1939 (if (or (not depth
) (< curdepth depth
))
1941 (setq curdepth
(1+ curdepth
))
1942 (setq ts
(cons (cons 'open-paren
1943 (cons (match-beginning 0) (match-end 0)))
1946 (cons 'semantic-list
1947 (cons (match-beginning 0)
1951 ;; This case makes flex robust
1956 semantic-flex-unterminated-syntax-end-function
1959 (setq ep
(point)))))
1962 ((looking-at "\\s)")
1963 (setq ts
(cons (cons 'close-paren
1964 (cons (match-beginning 0) (match-end 0)))
1966 (setq curdepth
(1- curdepth
)))
1967 ;; String initiators
1968 ((looking-at "\\s\"")
1969 ;; Zing to the end of this string.
1970 (setq ts
(cons (cons 'string
1971 (cons (match-beginning 0)
1975 ;; This case makes flex
1976 ;; robust to broken strings.
1980 semantic-flex-unterminated-syntax-end-function
1983 (setq ep
(point)))))
1987 (if (and semantic-ignore-comments
1988 (not semantic-flex-enable-whitespace
))
1989 ;; If the language doesn't deal with comments nor
1990 ;; whitespaces, ignore them here.
1991 (let ((comment-start-point (point)))
1993 (if (eq (point) comment-start-point
)
1994 ;; In this case our start-skip string failed
1995 ;; to work properly. Lets try and move over
1996 ;; whatever white space we matched to begin
1998 (skip-syntax-forward "-.'" (point-at-eol))
1999 ;;(forward-comment 1)
2000 ;; Generate newline token if enabled
2001 (if (and semantic-flex-enable-newlines
2004 (if (eq (point) comment-start-point
)
2005 (error "Strange comment syntax prevents lexical analysis"))
2007 (let ((tk (if semantic-ignore-comments
'whitespace
'comment
)))
2010 ;; Generate newline token if enabled
2011 (if (and semantic-flex-enable-newlines
2015 ;; Language wants comments or want them as whitespaces,
2016 ;; link them together.
2017 (if (eq (car (car ts
)) tk
)
2018 (setcdr (cdr (car ts
)) ep
)
2019 (setq ts
(cons (cons tk
(cons (match-beginning 0) ep
))
2022 ((looking-at "\\(\\s.\\|\\s$\\|\\s'\\)")
2023 (setq ts
(cons (cons 'punctuation
2024 (cons (match-beginning 0) (match-end 0)))
2028 (error "What is that?")))
2029 (goto-char (or ep
(match-end 0)))
2031 ;; maybe catch the last beginning of line when needed
2032 (and semantic-flex-enable-bol
2035 (setq ts
(cons (cons 'bol
(cons (point) (point))) ts
)))
2037 ;;(message "Flexing muscles...done")
2040 (provide 'semantic
/lex
)
2043 ;; generated-autoload-file: "loaddefs.el"
2044 ;; generated-autoload-load-name: "semantic/lex"
2047 ;;; semantic/lex.el ends here