1 ;;; lex-spp.el --- Semantic Lexical Pre-processor
3 ;;; Copyright (C) 2006, 2007, 2008, 2009 Free Software Foundation, Inc.
5 ;; Author: Eric M. Ludlam <zappo@gnu.org>
7 ;; This file is part of GNU Emacs.
9 ;; GNU Emacs is free software: you can redistribute it and/or modify
10 ;; it under the terms of the GNU General Public License as published by
11 ;; the Free Software Foundation, either version 3 of the License, or
12 ;; (at your option) any later version.
14 ;; GNU Emacs is distributed in the hope that it will be useful,
15 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
16 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 ;; GNU General Public License for more details.
19 ;; You should have received a copy of the GNU General Public License
20 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
24 ;; The Semantic Preprocessor works with semantic-lex to provide a phase
25 ;; during lexical analysis to do the work of a pre-processor.
27 ;; A pre-processor identifies lexical syntax mixed in with another language
28 ;; and replaces some keyword tokens with streams of alternate tokens.
30 ;; If you use SPP in your language, be sure to specify this in your
31 ;; semantic language setup function:
33 ;; (add-hook 'semantic-lex-reset-hooks 'semantic-lex-spp-reset-hook nil t)
36 ;; Special Lexical Tokens:
38 ;; There are several special lexical tokens that are used by the
39 ;; Semantic PreProcessor lexer. They are:
42 ;; spp-macro-def - A definition of a lexical macro.
43 ;; spp-macro-undef - A removal of a definition of a lexical macro.
44 ;; spp-system-include - A system level include file
45 ;; spp-include - An include file
46 ;; spp-concat - A lexical token representing textual concatenation
49 ;; Operational tokens:
50 ;; spp-arg-list - Represents an argument list to a macro.
51 ;; spp-symbol-merge - A request for multiple symbols to be textually merged.
55 ;; Use `semantic-push-parser-warning' for situations where there are likely
56 ;; macros that are undefined unexpectedly, or other problem.
60 ;; Try to handle the case of:
62 ;; #define NN namespace nn {
70 (require 'semantic
/lex
)
73 (defvar semantic-lex-spp-macro-symbol-obarray nil
74 "Table of macro keywords used by the Semantic Preprocessor.
75 These symbols will be used in addition to those in
76 `semantic-lex-spp-dynamic-macro-symbol-obarray'.")
77 (make-variable-buffer-local 'semantic-lex-spp-macro-symbol-obarray
)
79 (defvar semantic-lex-spp-project-macro-symbol-obarray nil
80 "Table of macro keywords for this project.
81 These symbols will be used in addition to those in
82 `semantic-lex-spp-dynamic-macro-symbol-obarray'.")
83 (make-variable-buffer-local 'semantic-lex-spp-project-macro-symbol-obarray
)
85 (defvar semantic-lex-spp-dynamic-macro-symbol-obarray nil
86 "Table of macro keywords used during lexical analysis.
87 Macros are lexical symbols which are replaced by other lexical
88 tokens during lexical analysis. During analysis symbols can be
89 added and removed from this symbol table.")
90 (make-variable-buffer-local 'semantic-lex-spp-dynamic-macro-symbol-obarray
)
92 (defvar semantic-lex-spp-dynamic-macro-symbol-obarray-stack nil
93 "A stack of obarrays for temporarilly scoped macro values.")
94 (make-variable-buffer-local 'semantic-lex-spp-dynamic-macro-symbol-obarray-stack
)
96 (defvar semantic-lex-spp-expanded-macro-stack nil
97 "The stack of lexical SPP macros we have expanded.")
98 ;; The above is not buffer local. Some macro expansions need to be
99 ;; dumped into a secondary buffer for re-lexing.
101 ;;; NON-RECURSIVE MACRO STACK
102 ;; C Pre-processor does not allow recursive macros. Here are some utils
103 ;; for managing the symbol stack of where we've been.
105 (defmacro semantic-lex-with-macro-used
(name &rest body
)
106 "With the macro NAME currently being expanded, execute BODY.
107 Pushes NAME into the macro stack. The above stack is checked
108 by `semantic-lex-spp-symbol' to not return true for any symbol
109 currently being expanded."
112 (push ,name semantic-lex-spp-expanded-macro-stack
)
114 (pop semantic-lex-spp-expanded-macro-stack
)))
115 (put 'semantic-lex-with-macro-used
'lisp-indent-function
1)
121 (def-edebug-spec semantic-lex-with-macro-used
127 ;;; MACRO TABLE UTILS
129 ;; The dynamic macro table is a buffer local variable that is modified
130 ;; during the analysis. OBARRAYs are used, so the language must
131 ;; have symbols that are compatible with Emacs Lisp symbols.
133 (defsubst semantic-lex-spp-symbol
(name)
134 "Return spp symbol with NAME or nil if not found.
135 The searcy priority is:
137 2. PROJECT specified symbols.
138 3. SYSTEM specified symbols."
142 ;; Make sure we don't recurse.
143 (not (member name semantic-lex-spp-expanded-macro-stack
))
144 ;; Do the check of the various tables.
147 (and (arrayp semantic-lex-spp-dynamic-macro-symbol-obarray
)
148 (intern-soft name semantic-lex-spp-dynamic-macro-symbol-obarray
))
150 (and (arrayp semantic-lex-spp-project-macro-symbol-obarray
)
151 (intern-soft name semantic-lex-spp-project-macro-symbol-obarray
))
153 (and (arrayp semantic-lex-spp-macro-symbol-obarray
)
154 (intern-soft name semantic-lex-spp-macro-symbol-obarray
))
158 (defsubst semantic-lex-spp-symbol-p
(name)
159 "Return non-nil if a keyword with NAME exists in any keyword table."
160 (if (semantic-lex-spp-symbol name
)
163 (defsubst semantic-lex-spp-dynamic-map
()
164 "Return the dynamic macro map for the current buffer."
165 (or semantic-lex-spp-dynamic-macro-symbol-obarray
166 (setq semantic-lex-spp-dynamic-macro-symbol-obarray
167 (make-vector 13 0))))
169 (defsubst semantic-lex-spp-dynamic-map-stack
()
170 "Return the dynamic macro map for the current buffer."
171 (or semantic-lex-spp-dynamic-macro-symbol-obarray-stack
172 (setq semantic-lex-spp-dynamic-macro-symbol-obarray-stack
173 (make-vector 13 0))))
175 (defun semantic-lex-spp-symbol-set (name value
&optional obarray-in
)
176 "Set value of spp symbol with NAME to VALUE and return VALUE.
177 If optional OBARRAY-IN is non-nil, then use that obarray instead of
179 (if (and (stringp value
) (string= value
"")) (setq value nil
))
180 (set (intern name
(or obarray-in
181 (semantic-lex-spp-dynamic-map)))
184 (defsubst semantic-lex-spp-symbol-remove
(name &optional obarray
)
185 "Remove the spp symbol with NAME.
186 If optional OBARRAY is non-nil, then use that obarray instead of
188 (unintern name
(or obarray
189 (semantic-lex-spp-dynamic-map))))
191 (defun semantic-lex-spp-symbol-push (name value
)
192 "Push macro NAME with VALUE into the map.
193 Reverse with `semantic-lex-spp-symbol-pop'."
194 (let* ((map (semantic-lex-spp-dynamic-map))
195 (stack (semantic-lex-spp-dynamic-map-stack))
196 (mapsym (intern name map
))
197 (stacksym (intern name stack
))
198 (mapvalue (when (boundp mapsym
) (symbol-value mapsym
)))
200 (when (boundp mapsym
)
201 ;; Make sure there is a stack
202 (if (not (boundp stacksym
)) (set stacksym nil
))
203 ;; If there is a value to push, then push it.
204 (set stacksym
(cons mapvalue
(symbol-value stacksym
)))
206 ;; Set our new value here.
210 (defun semantic-lex-spp-symbol-pop (name)
211 "Pop macro NAME from the stackmap into the orig map.
212 Reverse with `semantic-lex-spp-symbol-pop'."
213 (let* ((map (semantic-lex-spp-dynamic-map))
214 (stack (semantic-lex-spp-dynamic-map-stack))
215 (mapsym (intern name map
))
216 (stacksym (intern name stack
))
219 (if (or (not (boundp stacksym
) )
220 (= (length (symbol-value stacksym
)) 0))
221 ;; Nothing to pop, remove it.
223 ;; If there is a value to pop, then add it to the map.
224 (set mapsym
(car (symbol-value stacksym
)))
225 (set stacksym
(cdr (symbol-value stacksym
)))
228 (defsubst semantic-lex-spp-symbol-stream
(name)
229 "Return replacement stream of macro with NAME."
230 (let ((spp (semantic-lex-spp-symbol name
)))
232 (symbol-value spp
))))
234 (defun semantic-lex-make-spp-table (specs)
235 "Convert spp macro list SPECS into an obarray and return it.
236 SPECS must be a list of (NAME . REPLACEMENT) elements, where:
238 NAME is the name of the spp macro symbol to define.
239 REPLACEMENT a string that would be substituted in for NAME."
241 ;; Create the symbol hash table
242 (let ((semantic-lex-spp-macro-symbol-obarray (make-vector 13 0))
244 ;; fill it with stuff
246 (setq spec
(car specs
)
248 (semantic-lex-spp-symbol-set
251 semantic-lex-spp-macro-symbol-obarray
))
252 semantic-lex-spp-macro-symbol-obarray
))
254 (defun semantic-lex-spp-save-table ()
255 "Return a list of spp macros and values.
256 The return list is meant to be saved in a semanticdb table."
258 (when (arrayp semantic-lex-spp-dynamic-macro-symbol-obarray
)
261 (setq macros
(cons (cons (symbol-name symbol
)
262 (symbol-value symbol
))
264 semantic-lex-spp-dynamic-macro-symbol-obarray
))
267 (defun semantic-lex-spp-macros ()
268 "Return a list of spp macros as Lisp symbols.
269 The value of each symbol is the replacement stream."
271 (when (arrayp semantic-lex-spp-macro-symbol-obarray
)
274 (setq macros
(cons symbol macros
)))
275 semantic-lex-spp-macro-symbol-obarray
))
276 (when (arrayp semantic-lex-spp-project-macro-symbol-obarray
)
279 (setq macros
(cons symbol macros
)))
280 semantic-lex-spp-project-macro-symbol-obarray
))
281 (when (arrayp semantic-lex-spp-dynamic-macro-symbol-obarray
)
284 (setq macros
(cons symbol macros
)))
285 semantic-lex-spp-dynamic-macro-symbol-obarray
))
288 (defun semantic-lex-spp-set-dynamic-table (new-entries)
289 "Set the dynamic symbol table to NEW-ENTRIES.
290 For use with semanticdb restoration of state."
291 (dolist (e new-entries
)
292 ;; Default obarray for below is the dynamic map.
293 (semantic-lex-spp-symbol-set (car e
) (cdr e
))))
295 (defun semantic-lex-spp-reset-hook (start end
)
296 "Reset anything needed by SPP for parsing.
297 In this case, reset the dynamic macro symbol table if
298 START is (point-min).
300 (when (= start
(point-min))
301 (setq semantic-lex-spp-dynamic-macro-symbol-obarray nil
302 semantic-lex-spp-dynamic-macro-symbol-obarray-stack nil
303 ;; This shouldn't not be nil, but reset just in case.
304 semantic-lex-spp-expanded-macro-stack nil
)
307 ;;; MACRO EXPANSION: Simple cases
309 ;; If a user fills in the table with simple strings, we can
310 ;; support that by converting them into tokens with the
311 ;; various analyzers that are available.
313 (defun semantic-lex-spp-extract-regex-and-compare (analyzer value
)
314 "Extract a regexp from an ANALYZER and use to match VALUE.
315 Return non-nil if it matches"
316 (let* ((condition (car analyzer
))
317 (regex (cond ((eq (car condition
) 'looking-at
)
322 (string-match regex value
))
325 (defun semantic-lex-spp-simple-macro-to-macro-stream (val beg end argvalues
)
326 "Convert lexical macro contents VAL into a macro expansion stream.
327 These are for simple macro expansions that a user may have typed in directly.
328 As such, we need to analyze the input text, to figure out what kind of real
329 lexical token we should be inserting in its place.
331 Argument VAL is the value of some macro to be converted into a stream.
332 BEG and END are the token bounds of the macro to be expanded
333 that will somehow gain a much longer token stream.
334 ARGVALUES are values for any arg list, or nil."
336 ;; We perform a replacement. Technically, this should
337 ;; be a full lexical step over the "val" string, but take
338 ;; a guess that its just a keyword or existing symbol.
340 ;; Probably a really bad idea. See how it goes.
341 ((semantic-lex-spp-extract-regex-and-compare
342 semantic-lex-symbol-or-keyword val
)
343 (semantic-lex-push-token
344 (semantic-lex-token (or (semantic-lex-keyword-p val
) 'symbol
)
348 ;; Ok, the rest of these are various types of syntax.
349 ;; Conveniences for users that type in their symbol table.
350 ((semantic-lex-spp-extract-regex-and-compare
351 semantic-lex-punctuation val
)
352 (semantic-lex-token 'punctuation beg end val
))
353 ((semantic-lex-spp-extract-regex-and-compare
354 semantic-lex-number val
)
355 (semantic-lex-token 'number beg end val
))
356 ((semantic-lex-spp-extract-regex-and-compare
357 semantic-lex-paren-or-list val
)
358 (semantic-lex-token 'semantic-list beg end val
))
359 ((semantic-lex-spp-extract-regex-and-compare
360 semantic-lex-string val
)
361 (semantic-lex-token 'string beg end val
))
365 ;;; MACRO EXPANSION : Lexical token replacement
367 ;; When substituting in a macro from a token stream of formatted
368 ;; semantic lex tokens, things can be much more complicated.
370 ;; Some macros have arguments that get set into the dynamic macro
371 ;; table during replacement.
373 ;; In general, the macro tokens are substituted into the regular
374 ;; token stream, but placed under the characters of the original
377 ;; Argument lists are saved as a lexical token at the beginning
378 ;; of a replacement value.
380 (defun semantic-lex-spp-one-token-to-txt (tok &optional blocktok
)
381 "Convert the token TOK into a string.
382 If TOK is made of multiple tokens, convert those to text. This
383 conversion is needed if a macro has a merge symbol in it that
384 combines the text of two previously distinct symbols. For
387 #define (a,b) a ## b;
389 If optional string BLOCKTOK matches the expanded value, then do not
390 continue processing recursively."
391 (let ((txt (semantic-lex-token-text tok
))
395 ;; Recursion prevention
396 ((and (stringp blocktok
) (string= txt blocktok
))
399 ((and (eq (car tok
) 'symbol
)
400 (setq sym
(semantic-lex-spp-symbol txt
))
401 (not (semantic-lex-spp-macro-with-args (symbol-value sym
)))
403 ;; Now that we have a symbol,
404 (let ((val (symbol-value sym
)))
406 ;; This is another lexical token.
409 (semantic-lex-spp-one-token-to-txt val txt
))
410 ;; This is a list of tokens.
413 (symbolp (car (car val
))))
414 (mapconcat (lambda (subtok)
415 (semantic-lex-spp-one-token-to-txt subtok
))
418 ;; If val is nil, that's probably wrong.
419 ;; Found a system header case where this was true.
421 ;; Debug wierd stuff.
429 (defun semantic-lex-spp-macro-with-args (val)
430 "If the macro value VAL has an argument list, return the arglist."
431 (when (and val
(consp val
) (consp (car val
))
432 (eq 'spp-arg-list
(car (car val
))))
433 (car (cdr (car val
)))))
435 (defun semantic-lex-spp-token-macro-to-macro-stream (val beg end argvalues
)
436 "Convert lexical macro contents VAL into a macro expansion stream.
437 Argument VAL is the value of some macro to be converted into a stream.
438 BEG and END are the token bounds of the macro to be expanded
439 that will somehow gain a much longer token stream.
440 ARGVALUES are values for any arg list, or nil.
441 See comments in code for information about how token streams are processed
442 and what valid VAL values are."
444 ;; A typical VAL value might be either a stream of tokens.
445 ;; Tokens saved into a macro stream always includes the text from the
446 ;; buffer, since the locations specified probably don't represent
447 ;; that text anymore, or even the same buffer.
449 ;; CASE 1: Simple token stream
451 ;; #define SUPER mysuper::
453 ;;((symbol "mysuper" 480 . 487)
454 ;; (punctuation ":" 487 . 488)
455 ;; (punctuation ":" 488 . 489))
457 ;; CASE 2: Token stream with argument list
459 ;; #define INT_FCN(name) int name (int in)
461 ;; ((spp-arg-list ("name") 558 . 564)
462 ;; (INT "int" 565 . 568)
463 ;; (symbol "name" 569 . 573)
464 ;; (semantic-list "(int in)" 574 . 582))
466 ;; In the second case, a macro with an argument list as the a rgs as the
469 ;; CASE 3: Symbol text merge
471 ;; #define TMP(a) foo_ ## a
473 ;; ((spp-arg-list ("a") 20 . 23)
474 ;; (spp-symbol-merge ((symbol "foo_" 24 . 28) (symbol "a" 32 . 33))
477 ;; Usually in conjunction with a macro with an argument, merging symbol
478 ;; parts is a way of fabricating new symbols from pieces inside the macro.
479 ;; These macros use `spp-symbol-merge' tokens whose TEXT part is another
480 ;; token stream. This sub-stream ought to consist of only 2 SYMBOL pieces,
481 ;; though I suppose keywords might be ok. The end result of this example
482 ;; merge symbol would be (symbol "foo_A" 24 . 33) where A is the symbol
483 ;; passed in from the arg list "a".
485 ;; CASE 4: Nested token streams
488 ;; #define BLA bla FOO(foo)
490 ;; ((INT "int" 82 . 85)
491 ;; (symbol "FOO" 86 . 89)
492 ;; (semantic-list "(foo)" 89 . 94))
494 ;; Nested token FOO shows up in the table of macros, and gets replace
495 ;; inline. This is the same as case 2.
497 (let ((arglist (semantic-lex-spp-macro-with-args val
))
502 ;; CASE 2: Dealing with the arg list.
504 ;; Skip the arg list.
507 ;; Push args into the replacement list.
508 (let ((AV argvalues
))
510 (let* ((argval (car AV
)))
512 (semantic-lex-spp-symbol-push A argval
)
513 (setq argalist
(cons (cons A argval
) argalist
))
514 (setq AV
(cdr AV
)))))
517 ;; Set val-tmp after stripping arguments.
520 ;; CASE 1: Push everything else onto the list.
521 ;; Once the arg list is stripped off, CASE 2 is the same
524 (setq v
(car val-tmp
))
525 (setq val-tmp
(cdr val-tmp
))
527 (let* (;; The text of the current lexical token.
529 ;; Try to convert txt into a macro declaration. If it is
530 ;; not a macro, use nil.
531 (txt-macro-or-nil (semantic-lex-spp-symbol txt
))
532 ;; If our current token is a macro, then pull off the argument
535 (when txt-macro-or-nil
536 (semantic-lex-spp-macro-with-args (symbol-value txt-macro-or-nil
)))
538 ;; We need to peek at the next token when testing for
539 ;; used macros with arg lists.
540 (next-tok-class (semantic-lex-token-class (car val-tmp
)))
544 ;; CASE 3: Merge symbols together.
545 ((eq (semantic-lex-token-class v
) 'spp-symbol-merge
)
546 ;; We need to merge the tokens in the 'text segement together,
547 ;; and produce a single symbol from it.
549 (mapconcat (lambda (tok)
550 (semantic-lex-spp-one-token-to-txt tok
))
553 (semantic-lex-push-token
554 (semantic-lex-token 'symbol beg end newsym
))
557 ;; CASE 2: Argument replacement. If a discovered symbol is in
558 ;; the active list of arguments, then we need to substitute
560 ((and (eq (semantic-lex-token-class v
) 'symbol
) txt-macro-or-nil
561 (or (and macro-and-args
(eq next-tok-class
'semantic-list
))
562 (not macro-and-args
))
567 (semantic-lex-spp-stream-for-arglist (car val-tmp
)))
568 ;; We used up these args. Pull from the stream.
569 (setq val-tmp
(cdr val-tmp
))
572 (semantic-lex-with-macro-used txt
573 ;; Don't recurse directly into this same fcn, because it is
574 ;; convenient to have plain string replacements too.
575 (semantic-lex-spp-macro-to-macro-stream
576 (symbol-value txt-macro-or-nil
)
580 ;; This is a HACK for the C parser. The 'macros text
581 ;; property is some storage so that the parser can do
582 ;; some C specific text manipulations.
583 ((eq (semantic-lex-token-class v
) 'semantic-list
)
584 ;; Push our arg list onto the semantic list.
586 (setq txt
(concat txt
)) ; Copy the text.
587 (put-text-property 0 1 'macros argalist txt
))
588 (semantic-lex-push-token
589 (semantic-lex-token (semantic-lex-token-class v
) beg end txt
))
592 ;; CASE 1: Just another token in the stream.
595 (semantic-lex-push-token
596 (semantic-lex-token (semantic-lex-token-class v
) beg end txt
))
600 ;; CASE 2: The arg list we pushed onto the symbol table
601 ;; must now be removed.
603 (semantic-lex-spp-symbol-pop A
))
608 ;; Used when token streams from different macros include eachother.
609 ;; Merged macro streams perform in place replacements.
611 (defun semantic-lex-spp-merge-streams (raw-stream)
612 "Merge elements from the RAW-STREAM together.
613 Handle spp-concat symbol concatenation.
614 Handle Nested macro replacements.
615 Return the cooked stream."
616 (let ((cooked-stream nil
))
619 (cond ((eq (semantic-lex-token-class (car raw-stream
)) 'spp-concat
)
620 ;; handle hashhash, by skipping it.
621 (setq raw-stream
(cdr raw-stream
))
622 ;; Now merge the symbols.
623 (let ((prev-tok (car cooked-stream
))
624 (next-tok (car raw-stream
)))
625 (setq cooked-stream
(cdr cooked-stream
))
626 (push (semantic-lex-token
628 (semantic-lex-token-start prev-tok
)
629 (semantic-lex-token-end next-tok
)
630 (list prev-tok next-tok
))
634 (push (car raw-stream
) cooked-stream
))
636 (setq raw-stream
(cdr raw-stream
))
639 (nreverse cooked-stream
))
644 ;; There are two types of expansion.
646 ;; 1. Expansion using a value made up of lexical tokens.
647 ;; 2. User input replacement from a plain string.
649 (defun semantic-lex-spp-macro-to-macro-stream (val beg end argvalues
)
650 "Convert lexical macro contents VAL into a macro expansion stream.
651 Argument VAL is the value of some macro to be converted into a stream.
652 BEG and END are the token bounds of the macro to be expanded
653 that will somehow gain a much longer token stream.
654 ARGVALUES are values for any arg list, or nil."
656 ;; If val is nil, then just skip it.
658 ;; If it is a token, then return that token rebuilt.
659 ((and (consp val
) (car val
) (symbolp (car val
)))
660 (semantic-lex-push-token
661 (semantic-lex-token (car val
) beg end
(semantic-lex-token-text val
))))
662 ;; Test for a token list.
663 ((and (consp val
) (consp (car val
)) (car (car val
))
664 (symbolp (car (car val
))))
665 (semantic-lex-spp-token-macro-to-macro-stream val beg end argvalues
))
666 ;; Test for miscellaneous strings.
668 (semantic-lex-spp-simple-macro-to-macro-stream val beg end argvalues
))
671 ;;; --------------------------------------------------------
678 ;; An analyser that will push tokens from a macro in place
679 ;; of the macro symbol.
681 (defun semantic-lex-spp-anlyzer-do-replace (sym val beg end
)
682 "Do the lexical replacement for SYM with VAL.
683 Argument BEG and END specify the bounds of SYM in the buffer."
685 (setq semantic-lex-end-point end
)
691 ;; Check for arguments.
692 (setq arg-in
(semantic-lex-spp-macro-with-args val
))
698 (semantic-lex-spp-one-token-and-move-for-macro
700 (setq end
(semantic-lex-token-end arg-parsed
))
702 (when (and (listp arg-parsed
) (eq (car arg-parsed
) 'semantic-list
))
704 ;; Use lex to split up the contents of the argument list.
705 (semantic-lex-spp-stream-for-arglist arg-parsed
)
709 ;; if we have something to sub in, then do it.
710 (semantic-lex-spp-macro-to-macro-stream val beg end arg-split
)
711 (setq semantic-lex-end-point end
)
715 (defvar semantic-lex-spp-replacements-enabled t
716 "Non-nil means do replacements when finding keywords.
717 Disable this only to prevent recursive expansion issues.")
719 (defun semantic-lex-spp-analyzer-push-tokens-for-symbol (str beg end
)
720 "Push lexical tokens for the symbol or keyword STR.
721 STR occurs in the current buffer between BEG and END."
725 ;; It is a macro. Prepare for a replacement.
726 ((and semantic-lex-spp-replacements-enabled
727 (semantic-lex-spp-symbol-p str
))
728 (setq sym
(semantic-lex-spp-symbol str
)
729 val
(symbol-value sym
)
732 (let ((semantic-lex-spp-expanded-macro-stack
733 semantic-lex-spp-expanded-macro-stack
))
735 (semantic-lex-with-macro-used str
736 ;; Do direct replacements of single value macros of macros.
737 ;; This solves issues with a macro containing one symbol that
738 ;; is another macro, and get arg lists passed around.
739 (while (and val
(consp val
)
740 (semantic-lex-token-p (car val
))
742 (eq (semantic-lex-token-class (car val
)) 'symbol
)
743 (semantic-lex-spp-symbol-p (semantic-lex-token-text (car val
)))
746 (setq str
(semantic-lex-token-text (car val
)))
747 (setq sym
(semantic-lex-spp-symbol str
)
748 val
(symbol-value sym
))
750 (setq count
(1+ count
))
751 ;; This prevents a different kind of recursion.
752 (push str semantic-lex-spp-expanded-macro-stack
)
755 (semantic-lex-spp-anlyzer-do-replace sym val beg end
))
760 ;; A regular keyword.
761 (semantic-lex-push-token
762 (semantic-lex-token (or (semantic-lex-keyword-p str
) 'symbol
)
766 (define-lex-regex-analyzer semantic-lex-spp-replace-or-symbol-or-keyword
767 "Like 'semantic-lex-symbol-or-keyword' plus preprocessor macro replacement."
769 (let ((str (match-string 0))
770 (beg (match-beginning 0))
772 (semantic-lex-spp-analyzer-push-tokens-for-symbol str beg end
)))
774 ;;; ANALYZERS FOR NEW MACROS
776 ;; These utilities and analyzer declaration function are for
777 ;; creating an analyzer which produces new macros in the macro table.
779 ;; There are two analyzers. One for new macros, and one for removing
782 (defun semantic-lex-spp-first-token-arg-list (token)
783 "If TOKEN is a semantic-list, turn it into a an SPP ARG LIST."
784 (when (and (consp token
)
785 (symbolp (car token
))
786 (eq 'semantic-list
(car token
)))
787 ;; Convert TOKEN in place.
788 (let ((argsplit (cedet-split-string (semantic-lex-token-text token
)
790 (setcar token
'spp-arg-list
)
791 (setcar (nthcdr 1 token
) argsplit
))
794 (defun semantic-lex-spp-one-token-and-move-for-macro (max)
795 "Lex up one token, and move to end of that token.
797 (let ((ans (semantic-lex (point) max
0 0)))
799 (progn (goto-char max
)
801 (when (> (semantic-lex-token-end (car ans
)) max
)
802 (let ((bounds (semantic-lex-token-bounds (car ans
))))
803 (setcdr bounds max
)))
804 (goto-char (semantic-lex-token-end (car ans
)))
808 (defun semantic-lex-spp-stream-for-arglist (token)
809 "Lex up the contents of the arglist TOKEN.
810 Parsing starts inside the parens, and ends at the end of TOKEN."
811 (let ((end (semantic-lex-token-end token
))
816 (if (stringp (nth 1 token
))
817 ;; If the 2nd part of the token is a string, then we have
818 ;; a token specifically extracted from a buffer. Possibly
819 ;; a different buffer. This means we need to do something
820 ;; nice to parse its contents.
821 (let ((txt (semantic-lex-token-text token
)))
822 (semantic-lex-spp-lex-text-string
823 (substring txt
1 (1- (length txt
)))))
825 ;; This part is like the original
826 (goto-char (semantic-lex-token-start token
))
827 ;; A cheat for going into the semantic list.
829 (setq fresh-toks
(semantic-lex-spp-stream-for-macro (1- end
)))
830 (dolist (tok fresh-toks
)
831 (when (memq (semantic-lex-token-class tok
) '(symbol semantic-list
))
832 (setq toks
(cons tok toks
))))
836 (defun semantic-lex-spp-lex-text-string (text)
837 "Lex the text string TEXT using the current buffer's state.
838 Use this to parse text extracted from a macro as if it came from
839 the current buffer. Since the lexer is designed to only work in
840 a buffer, we need to create a new buffer, and populate it with rules
841 and variable state from the current buffer."
842 ;; @TODO - will this fcn recurse?
843 (let* ((buf (get-buffer-create " *SPP parse hack*"))
847 (origbuff (current-buffer))
848 (important-vars '(semantic-lex-spp-macro-symbol-obarray
849 semantic-lex-spp-project-macro-symbol-obarray
850 semantic-lex-spp-dynamic-macro-symbol-obarray
851 semantic-lex-spp-dynamic-macro-symbol-obarray-stack
852 semantic-lex-spp-expanded-macro-stack
858 ;; Below is a painful hack to make sure everything is setup correctly.
859 (when (not (eq major-mode mode
))
861 ;; Hack in mode-local
862 (activate-mode-local-bindings)
863 ;; CHEATER! The following 3 lines are from
864 ;; `semantic-new-buffer-fcn', but we don't want to turn
865 ;; on all the other annoying modes for this little task.
866 (setq semantic-new-buffer-fcn-was-run t
)
868 (semantic-clear-toplevel-cache)
869 (remove-hook 'semantic-lex-reset-hooks
'semantic-lex-spp-reset-hook
873 ;; Second Cheat: copy key variables reguarding macro state from the
874 ;; the originating buffer we are parsing. We need to do this every time
875 ;; since the state changes.
876 (dolist (V important-vars
)
877 (set V
(semantic-buffer-local-value V origbuff
)))
879 (goto-char (point-min))
881 (setq fresh-toks
(semantic-lex-spp-stream-for-macro (point-max))))
883 (dolist (tok fresh-toks
)
884 (when (memq (semantic-lex-token-class tok
) '(symbol semantic-list
))
885 (setq toks
(cons tok toks
))))
890 ;; This is the fist version of semantic-lex-spp-stream-for-arglist
891 ;; that worked pretty well. It doesn't work if the TOKEN was derived
892 ;; from some other buffer, in which case it can get the wrong answer
893 ;; or throw an error if the token location in the originating buffer is
894 ;; larger than the current buffer.
895 ;;(defun semantic-lex-spp-stream-for-arglist-orig (token)
896 ;; "Lex up the contents of the arglist TOKEN.
897 ;; Parsing starts inside the parens, and ends at the end of TOKEN."
899 ;; (let ((end (semantic-lex-token-end token))
902 ;; (goto-char (semantic-lex-token-start token))
903 ;; ;; A cheat for going into the semantic list.
905 ;; (setq fresh-toks (semantic-lex-spp-stream-for-macro (1- end)))
906 ;; (dolist (tok fresh-toks)
907 ;; (when (memq (semantic-lex-token-class tok) '(symbol semantic-list))
908 ;; (setq toks (cons tok toks))))
913 ;; This doesn't work, because some arguments passed into a macro
914 ;; might contain non-simple symbol words, which this doesn't handle.
916 ;; Thus, you need a full lex to occur.
917 ;; (defun semantic-lex-spp-stream-for-arglist-split (token)
918 ;; "Lex up the contents of the arglist TOKEN.
919 ;; Parsing starts inside the parens, and ends at the end of TOKEN."
920 ;; (let* ((txt (semantic-lex-token-text token))
921 ;; (split (split-string (substring txt 1 (1- (length txt)))
923 ;; ;; Hack for lexing.
924 ;; (semantic-lex-spp-analyzer-push-tokens-for-symbol nil))
926 ;; (semantic-lex-spp-analyzer-push-tokens-for-symbol S 0 1))
927 ;; (reverse semantic-lex-spp-analyzer-push-tokens-for-symbol)))
930 (defun semantic-lex-spp-stream-for-macro (eos)
931 "Lex up a stream of tokens for a #define statement.
932 Parsing starts at the current point location.
933 EOS is the end of the stream to lex for this macro."
935 (while (< (point) eos
)
936 (let* ((tok (semantic-lex-spp-one-token-and-move-for-macro eos
))
938 (semantic-lex-token-text tok
)))
941 (push (semantic-lex-token (semantic-lex-token-class tok
)
942 (semantic-lex-token-start tok
)
943 (semantic-lex-token-end tok
)
953 (defmacro define-lex-spp-macro-declaration-analyzer
(name doc regexp tokidx
955 "Define a lexical analyzer for defining new MACROS.
956 NAME is the name of the analyzer.
957 DOC is the documentation for the analyzer.
958 REGEXP is a regular expression for the analyzer to match.
959 See `define-lex-regex-analyzer' for more on regexp.
960 TOKIDX is an index into REGEXP for which a new lexical token
961 of type `spp-macro-def' is to be created.
962 VALFORM are forms that return the value to be saved for this macro, or nil.
963 When implementing a macro, you can use `semantic-lex-spp-stream-for-macro'
964 to convert text into a lexical stream for storage in the macro."
965 (let ((start (make-symbol "start"))
966 (end (make-symbol "end"))
967 (val (make-symbol "val"))
968 (startpnt (make-symbol "startpnt"))
969 (endpnt (make-symbol "endpnt")))
970 `(define-lex-regex-analyzer ,name
973 (let ((,start
(match-beginning ,tokidx
))
974 (,end
(match-end ,tokidx
))
975 (,startpnt semantic-lex-end-point
)
976 (,val
(save-match-data ,@valform
))
977 (,endpnt semantic-lex-end-point
))
978 (semantic-lex-spp-symbol-set
979 (buffer-substring-no-properties ,start
,end
)
981 (semantic-lex-push-token
982 (semantic-lex-token 'spp-macro-def
984 ;; Preserve setting of the end point from the calling macro.
985 (when (and (/= ,startpnt
,endpnt
)
986 (/= ,endpnt semantic-lex-end-point
))
987 (setq semantic-lex-end-point
,endpnt
))
990 (defmacro define-lex-spp-macro-undeclaration-analyzer
(name doc regexp tokidx
)
991 "Undefine a lexical analyzer for defining new MACROS.
992 NAME is the name of the analyzer.
993 DOC is the documentation for the analyzer.
994 REGEXP is a regular expression for the analyzer to match.
995 See `define-lex-regex-analyzer' for more on regexp.
996 TOKIDX is an index into REGEXP for which a new lexical token
997 of type `spp-macro-undef' is to be created."
998 (let ((start (make-symbol "start"))
999 (end (make-symbol "end")))
1000 `(define-lex-regex-analyzer ,name
1003 (let ((,start
(match-beginning ,tokidx
))
1004 (,end
(match-end ,tokidx
))
1006 (semantic-lex-spp-symbol-remove
1007 (buffer-substring-no-properties ,start
,end
))
1008 (semantic-lex-push-token
1009 (semantic-lex-token 'spp-macro-undef
1015 ;; These analyzers help a language define how include files
1016 ;; are identified. These are ONLY for languages that perform
1017 ;; an actual textual includesion, and not for imports.
1019 ;; This section is supposed to allow the macros from the headers to be
1020 ;; added to the local dynamic macro table, but that hasn't been
1023 (defcustom semantic-lex-spp-use-headers-flag nil
1024 "*Non-nil means to pre-parse headers as we go.
1025 For languages that use the Semantic pre-processor, this can
1026 improve the accuracy of parsed files where include files
1027 can change the state of what's parsed in the current file.
1029 Note: Note implemented yet"
1033 (defun semantic-lex-spp-merge-header (name)
1034 "Extract and merge any macros from the header with NAME.
1035 Finds the header file belonging to NAME, gets the macros
1036 from that file, and then merge the macros with our current
1038 (when semantic-lex-spp-use-headers-flag
1039 ;; @todo - do this someday, ok?
1042 (defmacro define-lex-spp-include-analyzer
(name doc regexp tokidx
1044 "Define a lexical analyzer for defining a new INCLUDE lexical token.
1045 Macros defined in the found include will be added to our running table
1046 at the time the include statement is found.
1047 NAME is the name of the analyzer.
1048 DOC is the documentation for the analyzer.
1049 REGEXP is a regular expression for the analyzer to match.
1050 See `define-lex-regex-analyzer' for more on regexp.
1051 TOKIDX is an index into REGEXP for which a new lexical token
1052 of type `spp-macro-include' is to be created.
1053 VALFORM are forms that return the name of the thing being included, and the
1054 type of include. The return value should be of the form:
1056 where NAME is the name of the include, and TYPE is the type of the include,
1057 where a valid symbol is 'system, or nil."
1058 (let ((start (make-symbol "start"))
1059 (end (make-symbol "end"))
1060 (val (make-symbol "val"))
1061 (startpnt (make-symbol "startpnt"))
1062 (endpnt (make-symbol "endpnt")))
1063 `(define-lex-regex-analyzer ,name
1066 (let ((,start
(match-beginning ,tokidx
))
1067 (,end
(match-end ,tokidx
))
1068 (,startpnt semantic-lex-end-point
)
1069 (,val
(save-match-data ,@valform
))
1070 (,endpnt semantic-lex-end-point
))
1071 ;;(message "(car ,val) -> %S" (car ,val))
1072 (semantic-lex-spp-merge-header (car ,val
))
1073 (semantic-lex-push-token
1074 (semantic-lex-token (if (eq (cdr ,val
) 'system
)
1079 ;; Preserve setting of the end point from the calling macro.
1080 (when (and (/= ,startpnt
,endpnt
)
1081 (/= ,endpnt semantic-lex-end-point
))
1082 (setq semantic-lex-end-point
,endpnt
))
1087 ;; Semanticdb can save off macro tables for quick lookup later.
1089 ;; These routines are for saving macro lists into an EIEIO persistent
1091 (defvar semantic-lex-spp-macro-max-length-to-save
200
1092 "*Maximum length of an SPP macro before we opt to not save it.")
1094 (defun semantic-lex-spp-table-write-slot-value (value)
1095 "Write out the VALUE of a slot for EIEIO.
1096 The VALUE is a spp lexical table."
1104 (let* ((first (car (cdr sym
)))
1106 (when (not (listp first
))
1107 (error "Error in macro \"%s\"" (car sym
)))
1108 (when (eq (car first
) 'spp-arg-list
)
1111 (setq rest
(cdr rest
))
1116 (let ((len (length (cdr rest
))))
1121 (princ "nil ;; Error writing macro\n"))))
1122 ((< len semantic-lex-spp-macro-max-length-to-save
)
1127 (princ "nil ;; Error writing macro\n ")))
1130 (princ "nil ;; Too Long!\n ")
1140 (defun semantic-lex-spp-write-test ()
1141 "Test the semantic tag writer against the current buffer."
1143 (with-output-to-temp-buffer "*SPP Write Test*"
1144 (semantic-lex-spp-table-write-slot-value
1145 (semantic-lex-spp-save-table))))
1147 (defun semantic-lex-spp-write-utest ()
1148 "Unit test using the test spp file to test the slot write fcn."
1150 (let* ((sem (locate-library "semantic-lex-spp.el"))
1151 (dir (file-name-directory sem
)))
1153 (set-buffer (find-file-noselect
1154 (expand-file-name "tests/testsppreplace.c"
1156 (semantic-lex-spp-write-test))))
1158 ;;; MACRO TABLE DEBUG
1160 (defun semantic-lex-spp-describe (&optional buffer
)
1161 "Describe the current list of spp macros for BUFFER.
1162 If BUFFER is not provided, use the current buffer."
1164 (let ((syms (save-excursion
1165 (if buffer
(set-buffer buffer
))
1166 (semantic-lex-spp-macros)))
1168 (with-output-to-temp-buffer "*SPP MACROS*"
1169 (princ "Macro\t\tValue\n")
1171 (setq sym
(car syms
)
1173 (princ (symbol-name sym
))
1175 (if (< (length (symbol-name sym
)) 8)
1177 (prin1 (symbol-value sym
))
1187 (def-edebug-spec define-lex-spp-macro-declaration-analyzer
1188 (&define name stringp stringp form def-body
)
1191 (def-edebug-spec define-lex-spp-macro-undeclaration-analyzer
1192 (&define name stringp stringp form
)
1195 (def-edebug-spec define-lex-spp-include-analyzer
1196 (&define name stringp stringp form def-body
)
1201 (provide 'semantic
/lex-spp
)
1203 ;;; semantic-lex-spp.el ends here