Synch to Eric M. Ludlam's upstream CEDET repository.
[bpt/emacs.git] / lisp / cedet / semantic / lex-spp.el
1 ;;; lex-spp.el --- Semantic Lexical Pre-processor
2
3 ;;; Copyright (C) 2006, 2007, 2008, 2009 Free Software Foundation, Inc.
4
5 ;; Author: Eric M. Ludlam <zappo@gnu.org>
6
7 ;; This file is part of GNU Emacs.
8
9 ;; GNU Emacs is free software: you can redistribute it and/or modify
10 ;; it under the terms of the GNU General Public License as published by
11 ;; the Free Software Foundation, either version 3 of the License, or
12 ;; (at your option) any later version.
13
14 ;; GNU Emacs is distributed in the hope that it will be useful,
15 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
16 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 ;; GNU General Public License for more details.
18
19 ;; You should have received a copy of the GNU General Public License
20 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
21
22 ;;; Commentary:
23 ;;
24 ;; The Semantic Preprocessor works with semantic-lex to provide a phase
25 ;; during lexical analysis to do the work of a pre-processor.
26 ;;
27 ;; A pre-processor identifies lexical syntax mixed in with another language
28 ;; and replaces some keyword tokens with streams of alternate tokens.
29 ;;
30 ;; If you use SPP in your language, be sure to specify this in your
31 ;; semantic language setup function:
32 ;;
33 ;; (add-hook 'semantic-lex-reset-hooks 'semantic-lex-spp-reset-hook nil t)
34 ;;
35 ;;
36 ;; Special Lexical Tokens:
37 ;;
38 ;; There are several special lexical tokens that are used by the
39 ;; Semantic PreProcessor lexer. They are:
40 ;;
41 ;; Declarations:
42 ;; spp-macro-def - A definition of a lexical macro.
43 ;; spp-macro-undef - A removal of a definition of a lexical macro.
44 ;; spp-system-include - A system level include file
45 ;; spp-include - An include file
46 ;; spp-concat - A lexical token representing textual concatenation
47 ;; of symbol parts.
48 ;;
49 ;; Operational tokens:
50 ;; spp-arg-list - Represents an argument list to a macro.
51 ;; spp-symbol-merge - A request for multiple symbols to be textually merged.
52 ;;
53 ;;; TODO:
54 ;;
55 ;; Use `semantic-push-parser-warning' for situations where there are likely
56 ;; macros that are undefined unexpectedly, or other problem.
57 ;;
58 ;; TODO:
59 ;;
60 ;; Try to handle the case of:
61 ;;
62 ;; #define NN namespace nn {
63 ;; #define NN_END }
64 ;;
65 ;; NN
66 ;; int mydecl() {}
67 ;; NN_END
68 ;;
69
70 (require 'semantic)
71 (require 'semantic/lex)
72
73 ;;; Code:
74 (defvar semantic-lex-spp-macro-symbol-obarray nil
75 "Table of macro keywords used by the Semantic Preprocessor.
76 These symbols will be used in addition to those in
77 `semantic-lex-spp-dynamic-macro-symbol-obarray'.")
78 (make-variable-buffer-local 'semantic-lex-spp-macro-symbol-obarray)
79
80 (defvar semantic-lex-spp-project-macro-symbol-obarray nil
81 "Table of macro keywords for this project.
82 These symbols will be used in addition to those in
83 `semantic-lex-spp-dynamic-macro-symbol-obarray'.")
84 (make-variable-buffer-local 'semantic-lex-spp-project-macro-symbol-obarray)
85
86 (defvar semantic-lex-spp-dynamic-macro-symbol-obarray nil
87 "Table of macro keywords used during lexical analysis.
88 Macros are lexical symbols which are replaced by other lexical
89 tokens during lexical analysis. During analysis symbols can be
90 added and removed from this symbol table.")
91 (make-variable-buffer-local 'semantic-lex-spp-dynamic-macro-symbol-obarray)
92
93 (defvar semantic-lex-spp-dynamic-macro-symbol-obarray-stack nil
94 "A stack of obarrays for temporarilly scoped macro values.")
95 (make-variable-buffer-local 'semantic-lex-spp-dynamic-macro-symbol-obarray-stack)
96
97 (defvar semantic-lex-spp-expanded-macro-stack nil
98 "The stack of lexical SPP macros we have expanded.")
99 ;; The above is not buffer local. Some macro expansions need to be
100 ;; dumped into a secondary buffer for re-lexing.
101
102 ;;; NON-RECURSIVE MACRO STACK
103 ;; C Pre-processor does not allow recursive macros. Here are some utils
104 ;; for managing the symbol stack of where we've been.
105
106 (defmacro semantic-lex-with-macro-used (name &rest body)
107 "With the macro NAME currently being expanded, execute BODY.
108 Pushes NAME into the macro stack. The above stack is checked
109 by `semantic-lex-spp-symbol' to not return true for any symbol
110 currently being expanded."
111 `(unwind-protect
112 (progn
113 (push ,name semantic-lex-spp-expanded-macro-stack)
114 ,@body)
115 (pop semantic-lex-spp-expanded-macro-stack)))
116 (put 'semantic-lex-with-macro-used 'lisp-indent-function 1)
117
118 (add-hook
119 'edebug-setup-hook
120 #'(lambda ()
121
122 (def-edebug-spec semantic-lex-with-macro-used
123 (symbolp def-body)
124 )
125
126 ))
127
128 ;;; MACRO TABLE UTILS
129 ;;
130 ;; The dynamic macro table is a buffer local variable that is modified
131 ;; during the analysis. OBARRAYs are used, so the language must
132 ;; have symbols that are compatible with Emacs Lisp symbols.
133 ;;
134 (defsubst semantic-lex-spp-symbol (name)
135 "Return spp symbol with NAME or nil if not found.
136 The searcy priority is:
137 1. DYNAMIC symbols
138 2. PROJECT specified symbols.
139 3. SYSTEM specified symbols."
140 (and
141 ;; Only strings...
142 (stringp name)
143 ;; Make sure we don't recurse.
144 (not (member name semantic-lex-spp-expanded-macro-stack))
145 ;; Do the check of the various tables.
146 (or
147 ;; DYNAMIC
148 (and (arrayp semantic-lex-spp-dynamic-macro-symbol-obarray)
149 (intern-soft name semantic-lex-spp-dynamic-macro-symbol-obarray))
150 ;; PROJECT
151 (and (arrayp semantic-lex-spp-project-macro-symbol-obarray)
152 (intern-soft name semantic-lex-spp-project-macro-symbol-obarray))
153 ;; SYSTEM
154 (and (arrayp semantic-lex-spp-macro-symbol-obarray)
155 (intern-soft name semantic-lex-spp-macro-symbol-obarray))
156 ;; ...
157 )))
158
159 (defsubst semantic-lex-spp-symbol-p (name)
160 "Return non-nil if a keyword with NAME exists in any keyword table."
161 (if (semantic-lex-spp-symbol name)
162 t))
163
164 (defsubst semantic-lex-spp-dynamic-map ()
165 "Return the dynamic macro map for the current buffer."
166 (or semantic-lex-spp-dynamic-macro-symbol-obarray
167 (setq semantic-lex-spp-dynamic-macro-symbol-obarray
168 (make-vector 13 0))))
169
170 (defsubst semantic-lex-spp-dynamic-map-stack ()
171 "Return the dynamic macro map for the current buffer."
172 (or semantic-lex-spp-dynamic-macro-symbol-obarray-stack
173 (setq semantic-lex-spp-dynamic-macro-symbol-obarray-stack
174 (make-vector 13 0))))
175
176 (defun semantic-lex-spp-symbol-set (name value &optional obarray-in)
177 "Set value of spp symbol with NAME to VALUE and return VALUE.
178 If optional OBARRAY-IN is non-nil, then use that obarray instead of
179 the dynamic map."
180 (if (and (stringp value) (string= value "")) (setq value nil))
181 (set (intern name (or obarray-in
182 (semantic-lex-spp-dynamic-map)))
183 value))
184
185 (defsubst semantic-lex-spp-symbol-remove (name &optional obarray)
186 "Remove the spp symbol with NAME.
187 If optional OBARRAY is non-nil, then use that obarray instead of
188 the dynamic map."
189 (unintern name (or obarray
190 (semantic-lex-spp-dynamic-map))))
191
192 (defun semantic-lex-spp-symbol-push (name value)
193 "Push macro NAME with VALUE into the map.
194 Reverse with `semantic-lex-spp-symbol-pop'."
195 (let* ((map (semantic-lex-spp-dynamic-map))
196 (stack (semantic-lex-spp-dynamic-map-stack))
197 (mapsym (intern name map))
198 (stacksym (intern name stack))
199 (mapvalue (when (boundp mapsym) (symbol-value mapsym)))
200 )
201 (when (boundp mapsym)
202 ;; Make sure there is a stack
203 (if (not (boundp stacksym)) (set stacksym nil))
204 ;; If there is a value to push, then push it.
205 (set stacksym (cons mapvalue (symbol-value stacksym)))
206 )
207 ;; Set our new value here.
208 (set mapsym value)
209 ))
210
211 (defun semantic-lex-spp-symbol-pop (name)
212 "Pop macro NAME from the stackmap into the orig map.
213 Reverse with `semantic-lex-spp-symbol-pop'."
214 (let* ((map (semantic-lex-spp-dynamic-map))
215 (stack (semantic-lex-spp-dynamic-map-stack))
216 (mapsym (intern name map))
217 (stacksym (intern name stack))
218 (oldvalue nil)
219 )
220 (if (or (not (boundp stacksym) )
221 (= (length (symbol-value stacksym)) 0))
222 ;; Nothing to pop, remove it.
223 (unintern name map)
224 ;; If there is a value to pop, then add it to the map.
225 (set mapsym (car (symbol-value stacksym)))
226 (set stacksym (cdr (symbol-value stacksym)))
227 )))
228
229 (defsubst semantic-lex-spp-symbol-stream (name)
230 "Return replacement stream of macro with NAME."
231 (let ((spp (semantic-lex-spp-symbol name)))
232 (if spp
233 (symbol-value spp))))
234
235 (defun semantic-lex-make-spp-table (specs)
236 "Convert spp macro list SPECS into an obarray and return it.
237 SPECS must be a list of (NAME . REPLACEMENT) elements, where:
238
239 NAME is the name of the spp macro symbol to define.
240 REPLACEMENT a string that would be substituted in for NAME."
241
242 ;; Create the symbol hash table
243 (let ((semantic-lex-spp-macro-symbol-obarray (make-vector 13 0))
244 spec)
245 ;; fill it with stuff
246 (while specs
247 (setq spec (car specs)
248 specs (cdr specs))
249 (semantic-lex-spp-symbol-set
250 (car spec)
251 (cdr spec)
252 semantic-lex-spp-macro-symbol-obarray))
253 semantic-lex-spp-macro-symbol-obarray))
254
255 (defun semantic-lex-spp-save-table ()
256 "Return a list of spp macros and values.
257 The return list is meant to be saved in a semanticdb table."
258 (let (macros)
259 (when (arrayp semantic-lex-spp-dynamic-macro-symbol-obarray)
260 (mapatoms
261 #'(lambda (symbol)
262 (setq macros (cons (cons (symbol-name symbol)
263 (symbol-value symbol))
264 macros)))
265 semantic-lex-spp-dynamic-macro-symbol-obarray))
266 macros))
267
268 (defun semantic-lex-spp-macros ()
269 "Return a list of spp macros as Lisp symbols.
270 The value of each symbol is the replacement stream."
271 (let (macros)
272 (when (arrayp semantic-lex-spp-macro-symbol-obarray)
273 (mapatoms
274 #'(lambda (symbol)
275 (setq macros (cons symbol macros)))
276 semantic-lex-spp-macro-symbol-obarray))
277 (when (arrayp semantic-lex-spp-project-macro-symbol-obarray)
278 (mapatoms
279 #'(lambda (symbol)
280 (setq macros (cons symbol macros)))
281 semantic-lex-spp-project-macro-symbol-obarray))
282 (when (arrayp semantic-lex-spp-dynamic-macro-symbol-obarray)
283 (mapatoms
284 #'(lambda (symbol)
285 (setq macros (cons symbol macros)))
286 semantic-lex-spp-dynamic-macro-symbol-obarray))
287 macros))
288
289 (defun semantic-lex-spp-set-dynamic-table (new-entries)
290 "Set the dynamic symbol table to NEW-ENTRIES.
291 For use with semanticdb restoration of state."
292 (dolist (e new-entries)
293 ;; Default obarray for below is the dynamic map.
294 (semantic-lex-spp-symbol-set (car e) (cdr e))))
295
296 (defun semantic-lex-spp-reset-hook (start end)
297 "Reset anything needed by SPP for parsing.
298 In this case, reset the dynamic macro symbol table if
299 START is (point-min).
300 END is not used."
301 (when (= start (point-min))
302 (setq semantic-lex-spp-dynamic-macro-symbol-obarray nil
303 semantic-lex-spp-dynamic-macro-symbol-obarray-stack nil
304 ;; This shouldn't not be nil, but reset just in case.
305 semantic-lex-spp-expanded-macro-stack nil)
306 ))
307
308 ;;; MACRO EXPANSION: Simple cases
309 ;;
310 ;; If a user fills in the table with simple strings, we can
311 ;; support that by converting them into tokens with the
312 ;; various analyzers that are available.
313
314 (defun semantic-lex-spp-extract-regex-and-compare (analyzer value)
315 "Extract a regexp from an ANALYZER and use to match VALUE.
316 Return non-nil if it matches"
317 (let* ((condition (car analyzer))
318 (regex (cond ((eq (car condition) 'looking-at)
319 (nth 1 condition))
320 (t
321 nil))))
322 (when regex
323 (string-match regex value))
324 ))
325
326 (defun semantic-lex-spp-simple-macro-to-macro-stream (val beg end argvalues)
327 "Convert lexical macro contents VAL into a macro expansion stream.
328 These are for simple macro expansions that a user may have typed in directly.
329 As such, we need to analyze the input text, to figure out what kind of real
330 lexical token we should be inserting in its place.
331
332 Argument VAL is the value of some macro to be converted into a stream.
333 BEG and END are the token bounds of the macro to be expanded
334 that will somehow gain a much longer token stream.
335 ARGVALUES are values for any arg list, or nil."
336 (cond
337 ;; We perform a replacement. Technically, this should
338 ;; be a full lexical step over the "val" string, but take
339 ;; a guess that its just a keyword or existing symbol.
340 ;;
341 ;; Probably a really bad idea. See how it goes.
342 ((semantic-lex-spp-extract-regex-and-compare
343 semantic-lex-symbol-or-keyword val)
344 (semantic-lex-push-token
345 (semantic-lex-token (or (semantic-lex-keyword-p val) 'symbol)
346 beg end
347 val)))
348
349 ;; Ok, the rest of these are various types of syntax.
350 ;; Conveniences for users that type in their symbol table.
351 ((semantic-lex-spp-extract-regex-and-compare
352 semantic-lex-punctuation val)
353 (semantic-lex-token 'punctuation beg end val))
354 ((semantic-lex-spp-extract-regex-and-compare
355 semantic-lex-number val)
356 (semantic-lex-token 'number beg end val))
357 ((semantic-lex-spp-extract-regex-and-compare
358 semantic-lex-paren-or-list val)
359 (semantic-lex-token 'semantic-list beg end val))
360 ((semantic-lex-spp-extract-regex-and-compare
361 semantic-lex-string val)
362 (semantic-lex-token 'string beg end val))
363 (t nil)
364 ))
365
366 ;;; MACRO EXPANSION : Lexical token replacement
367 ;;
368 ;; When substituting in a macro from a token stream of formatted
369 ;; semantic lex tokens, things can be much more complicated.
370 ;;
371 ;; Some macros have arguments that get set into the dynamic macro
372 ;; table during replacement.
373 ;;
374 ;; In general, the macro tokens are substituted into the regular
375 ;; token stream, but placed under the characters of the original
376 ;; macro symbol.
377 ;;
378 ;; Argument lists are saved as a lexical token at the beginning
379 ;; of a replacement value.
380
381 (defun semantic-lex-spp-one-token-to-txt (tok &optional blocktok)
382 "Convert the token TOK into a string.
383 If TOK is made of multiple tokens, convert those to text. This
384 conversion is needed if a macro has a merge symbol in it that
385 combines the text of two previously distinct symbols. For
386 exampe, in c:
387
388 #define (a,b) a ## b;
389
390 If optional string BLOCKTOK matches the expanded value, then do not
391 continue processing recursively."
392 (let ((txt (semantic-lex-token-text tok))
393 (sym nil)
394 )
395 (cond
396 ;; Recursion prevention
397 ((and (stringp blocktok) (string= txt blocktok))
398 blocktok)
399 ;; A complex symbol
400 ((and (eq (car tok) 'symbol)
401 (setq sym (semantic-lex-spp-symbol txt))
402 (not (semantic-lex-spp-macro-with-args (symbol-value sym)))
403 )
404 ;; Now that we have a symbol,
405 (let ((val (symbol-value sym)))
406 (cond
407 ;; This is another lexical token.
408 ((and (consp val)
409 (symbolp (car val)))
410 (semantic-lex-spp-one-token-to-txt val txt))
411 ;; This is a list of tokens.
412 ((and (consp val)
413 (consp (car val))
414 (symbolp (car (car val))))
415 (mapconcat (lambda (subtok)
416 (semantic-lex-spp-one-token-to-txt subtok))
417 val
418 ""))
419 ;; If val is nil, that's probably wrong.
420 ;; Found a system header case where this was true.
421 ((null val) "")
422 ;; Debug wierd stuff.
423 (t (debug)))
424 ))
425 ((stringp txt)
426 txt)
427 (t nil))
428 ))
429
430 (defun semantic-lex-spp-macro-with-args (val)
431 "If the macro value VAL has an argument list, return the arglist."
432 (when (and val (consp val) (consp (car val))
433 (eq 'spp-arg-list (car (car val))))
434 (car (cdr (car val)))))
435
436 (defun semantic-lex-spp-token-macro-to-macro-stream (val beg end argvalues)
437 "Convert lexical macro contents VAL into a macro expansion stream.
438 Argument VAL is the value of some macro to be converted into a stream.
439 BEG and END are the token bounds of the macro to be expanded
440 that will somehow gain a much longer token stream.
441 ARGVALUES are values for any arg list, or nil.
442 See comments in code for information about how token streams are processed
443 and what valid VAL values are."
444
445 ;; A typical VAL value might be either a stream of tokens.
446 ;; Tokens saved into a macro stream always includes the text from the
447 ;; buffer, since the locations specified probably don't represent
448 ;; that text anymore, or even the same buffer.
449 ;;
450 ;; CASE 1: Simple token stream
451 ;;
452 ;; #define SUPER mysuper::
453 ;; ==>
454 ;;((symbol "mysuper" 480 . 487)
455 ;; (punctuation ":" 487 . 488)
456 ;; (punctuation ":" 488 . 489))
457 ;;
458 ;; CASE 2: Token stream with argument list
459 ;;
460 ;; #define INT_FCN(name) int name (int in)
461 ;; ==>
462 ;; ((spp-arg-list ("name") 558 . 564)
463 ;; (INT "int" 565 . 568)
464 ;; (symbol "name" 569 . 573)
465 ;; (semantic-list "(int in)" 574 . 582))
466 ;;
467 ;; In the second case, a macro with an argument list as the a rgs as the
468 ;; first entry.
469 ;;
470 ;; CASE 3: Symbol text merge
471 ;;
472 ;; #define TMP(a) foo_ ## a
473 ;; ==>
474 ;; ((spp-arg-list ("a") 20 . 23)
475 ;; (spp-symbol-merge ((symbol "foo_" 24 . 28) (symbol "a" 32 . 33))
476 ;; 24 . 33))
477 ;;
478 ;; Usually in conjunction with a macro with an argument, merging symbol
479 ;; parts is a way of fabricating new symbols from pieces inside the macro.
480 ;; These macros use `spp-symbol-merge' tokens whose TEXT part is another
481 ;; token stream. This sub-stream ought to consist of only 2 SYMBOL pieces,
482 ;; though I suppose keywords might be ok. The end result of this example
483 ;; merge symbol would be (symbol "foo_A" 24 . 33) where A is the symbol
484 ;; passed in from the arg list "a".
485 ;;
486 ;; CASE 4: Nested token streams
487 ;;
488 ;; #define FOO(f) f
489 ;; #define BLA bla FOO(foo)
490 ;; ==>
491 ;; ((INT "int" 82 . 85)
492 ;; (symbol "FOO" 86 . 89)
493 ;; (semantic-list "(foo)" 89 . 94))
494 ;;
495 ;; Nested token FOO shows up in the table of macros, and gets replace
496 ;; inline. This is the same as case 2.
497
498 (let ((arglist (semantic-lex-spp-macro-with-args val))
499 (argalist nil)
500 (val-tmp nil)
501 (v nil)
502 )
503 ;; CASE 2: Dealing with the arg list.
504 (when arglist
505 ;; Skip the arg list.
506 (setq val (cdr val))
507
508 ;; Push args into the replacement list.
509 (let ((AV argvalues))
510 (dolist (A arglist)
511 (let* ((argval (car AV)))
512
513 (semantic-lex-spp-symbol-push A argval)
514 (setq argalist (cons (cons A argval) argalist))
515 (setq AV (cdr AV)))))
516 )
517
518 ;; Set val-tmp after stripping arguments.
519 (setq val-tmp val)
520
521 ;; CASE 1: Push everything else onto the list.
522 ;; Once the arg list is stripped off, CASE 2 is the same
523 ;; as CASE 1.
524 (while val-tmp
525 (setq v (car val-tmp))
526 (setq val-tmp (cdr val-tmp))
527
528 (let* (;; The text of the current lexical token.
529 (txt (car (cdr v)))
530 ;; Try to convert txt into a macro declaration. If it is
531 ;; not a macro, use nil.
532 (txt-macro-or-nil (semantic-lex-spp-symbol txt))
533 ;; If our current token is a macro, then pull off the argument
534 ;; list.
535 (macro-and-args
536 (when txt-macro-or-nil
537 (semantic-lex-spp-macro-with-args (symbol-value txt-macro-or-nil)))
538 )
539 ;; We need to peek at the next token when testing for
540 ;; used macros with arg lists.
541 (next-tok-class (semantic-lex-token-class (car val-tmp)))
542 )
543
544 (cond
545 ;; CASE 3: Merge symbols together.
546 ((eq (semantic-lex-token-class v) 'spp-symbol-merge)
547 ;; We need to merge the tokens in the 'text segement together,
548 ;; and produce a single symbol from it.
549 (let ((newsym
550 (mapconcat (lambda (tok)
551 (semantic-lex-spp-one-token-to-txt tok))
552 txt
553 "")))
554 (semantic-lex-push-token
555 (semantic-lex-token 'symbol beg end newsym))
556 ))
557
558 ;; CASE 2: Argument replacement. If a discovered symbol is in
559 ;; the active list of arguments, then we need to substitute
560 ;; in the new value.
561 ((and (eq (semantic-lex-token-class v) 'symbol) txt-macro-or-nil
562 (or (and macro-and-args (eq next-tok-class 'semantic-list))
563 (not macro-and-args))
564 )
565 (let ((AV nil))
566 (when macro-and-args
567 (setq AV
568 (semantic-lex-spp-stream-for-arglist (car val-tmp)))
569 ;; We used up these args. Pull from the stream.
570 (setq val-tmp (cdr val-tmp))
571 )
572
573 (semantic-lex-with-macro-used txt
574 ;; Don't recurse directly into this same fcn, because it is
575 ;; convenient to have plain string replacements too.
576 (semantic-lex-spp-macro-to-macro-stream
577 (symbol-value txt-macro-or-nil)
578 beg end AV))
579 ))
580
581 ;; This is a HACK for the C parser. The 'macros text
582 ;; property is some storage so that the parser can do
583 ;; some C specific text manipulations.
584 ((eq (semantic-lex-token-class v) 'semantic-list)
585 ;; Push our arg list onto the semantic list.
586 (when argalist
587 (setq txt (concat txt)) ; Copy the text.
588 (put-text-property 0 1 'macros argalist txt))
589 (semantic-lex-push-token
590 (semantic-lex-token (semantic-lex-token-class v) beg end txt))
591 )
592
593 ;; CASE 1: Just another token in the stream.
594 (t
595 ;; Nothing new.
596 (semantic-lex-push-token
597 (semantic-lex-token (semantic-lex-token-class v) beg end txt))
598 )
599 )))
600
601 ;; CASE 2: The arg list we pushed onto the symbol table
602 ;; must now be removed.
603 (dolist (A arglist)
604 (semantic-lex-spp-symbol-pop A))
605 ))
606
607 ;;; Macro Merging
608 ;;
609 ;; Used when token streams from different macros include eachother.
610 ;; Merged macro streams perform in place replacements.
611
612 (defun semantic-lex-spp-merge-streams (raw-stream)
613 "Merge elements from the RAW-STREAM together.
614 Handle spp-concat symbol concatenation.
615 Handle Nested macro replacements.
616 Return the cooked stream."
617 (let ((cooked-stream nil))
618 ;; Merge the stream
619 (while raw-stream
620 (cond ((eq (semantic-lex-token-class (car raw-stream)) 'spp-concat)
621 ;; handle hashhash, by skipping it.
622 (setq raw-stream (cdr raw-stream))
623 ;; Now merge the symbols.
624 (let ((prev-tok (car cooked-stream))
625 (next-tok (car raw-stream)))
626 (setq cooked-stream (cdr cooked-stream))
627 (push (semantic-lex-token
628 'spp-symbol-merge
629 (semantic-lex-token-start prev-tok)
630 (semantic-lex-token-end next-tok)
631 (list prev-tok next-tok))
632 cooked-stream)
633 ))
634 (t
635 (push (car raw-stream) cooked-stream))
636 )
637 (setq raw-stream (cdr raw-stream))
638 )
639
640 (nreverse cooked-stream))
641 )
642
643 ;;; MACRO EXPANSION
644 ;;
645 ;; There are two types of expansion.
646 ;;
647 ;; 1. Expansion using a value made up of lexical tokens.
648 ;; 2. User input replacement from a plain string.
649
650 (defun semantic-lex-spp-macro-to-macro-stream (val beg end argvalues)
651 "Convert lexical macro contents VAL into a macro expansion stream.
652 Argument VAL is the value of some macro to be converted into a stream.
653 BEG and END are the token bounds of the macro to be expanded
654 that will somehow gain a much longer token stream.
655 ARGVALUES are values for any arg list, or nil."
656 (cond
657 ;; If val is nil, then just skip it.
658 ((null val) t)
659 ;; If it is a token, then return that token rebuilt.
660 ((and (consp val) (car val) (symbolp (car val)))
661 (semantic-lex-push-token
662 (semantic-lex-token (car val) beg end (semantic-lex-token-text val))))
663 ;; Test for a token list.
664 ((and (consp val) (consp (car val)) (car (car val))
665 (symbolp (car (car val))))
666 (semantic-lex-spp-token-macro-to-macro-stream val beg end argvalues))
667 ;; Test for miscellaneous strings.
668 ((stringp val)
669 (semantic-lex-spp-simple-macro-to-macro-stream val beg end argvalues))
670 ))
671
672 ;;; --------------------------------------------------------
673 ;;;
674 ;;; ANALYZERS:
675 ;;;
676
677 ;;; Symbol Is Macro
678 ;;
679 ;; An analyser that will push tokens from a macro in place
680 ;; of the macro symbol.
681 ;;
682 (defun semantic-lex-spp-anlyzer-do-replace (sym val beg end)
683 "Do the lexical replacement for SYM with VAL.
684 Argument BEG and END specify the bounds of SYM in the buffer."
685 (if (not val)
686 (setq semantic-lex-end-point end)
687 (let ((arg-in nil)
688 (arg-parsed nil)
689 (arg-split nil)
690 )
691
692 ;; Check for arguments.
693 (setq arg-in (semantic-lex-spp-macro-with-args val))
694
695 (when arg-in
696 (save-excursion
697 (goto-char end)
698 (setq arg-parsed
699 (semantic-lex-spp-one-token-and-move-for-macro
700 (point-at-eol)))
701 (setq end (semantic-lex-token-end arg-parsed))
702
703 (when (and (listp arg-parsed) (eq (car arg-parsed) 'semantic-list))
704 (setq arg-split
705 ;; Use lex to split up the contents of the argument list.
706 (semantic-lex-spp-stream-for-arglist arg-parsed)
707 ))
708 ))
709
710 ;; if we have something to sub in, then do it.
711 (semantic-lex-spp-macro-to-macro-stream val beg end arg-split)
712 (setq semantic-lex-end-point end)
713 )
714 ))
715
716 (defvar semantic-lex-spp-replacements-enabled t
717 "Non-nil means do replacements when finding keywords.
718 Disable this only to prevent recursive expansion issues.")
719
720 (defun semantic-lex-spp-analyzer-push-tokens-for-symbol (str beg end)
721 "Push lexical tokens for the symbol or keyword STR.
722 STR occurs in the current buffer between BEG and END."
723 (let (sym val count)
724 (cond
725 ;;
726 ;; It is a macro. Prepare for a replacement.
727 ((and semantic-lex-spp-replacements-enabled
728 (semantic-lex-spp-symbol-p str))
729 (setq sym (semantic-lex-spp-symbol str)
730 val (symbol-value sym)
731 count 0)
732
733 (let ((semantic-lex-spp-expanded-macro-stack
734 semantic-lex-spp-expanded-macro-stack))
735
736 (semantic-lex-with-macro-used str
737 ;; Do direct replacements of single value macros of macros.
738 ;; This solves issues with a macro containing one symbol that
739 ;; is another macro, and get arg lists passed around.
740 (while (and val (consp val)
741 (semantic-lex-token-p (car val))
742 (eq (length val) 1)
743 (eq (semantic-lex-token-class (car val)) 'symbol)
744 (semantic-lex-spp-symbol-p (semantic-lex-token-text (car val)))
745 (< count 10)
746 )
747 (setq str (semantic-lex-token-text (car val)))
748 (setq sym (semantic-lex-spp-symbol str)
749 val (symbol-value sym))
750 ;; Prevent recursion
751 (setq count (1+ count))
752 ;; This prevents a different kind of recursion.
753 (push str semantic-lex-spp-expanded-macro-stack)
754 )
755
756 (semantic-lex-spp-anlyzer-do-replace sym val beg end))
757
758 ))
759 ;; Anything else.
760 (t
761 ;; A regular keyword.
762 (semantic-lex-push-token
763 (semantic-lex-token (or (semantic-lex-keyword-p str) 'symbol)
764 beg end))))
765 ))
766
767 (define-lex-regex-analyzer semantic-lex-spp-replace-or-symbol-or-keyword
768 "Like 'semantic-lex-symbol-or-keyword' plus preprocessor macro replacement."
769 "\\(\\sw\\|\\s_\\)+"
770 (let ((str (match-string 0))
771 (beg (match-beginning 0))
772 (end (match-end 0)))
773 (semantic-lex-spp-analyzer-push-tokens-for-symbol str beg end)))
774
775 ;;; ANALYZERS FOR NEW MACROS
776 ;;
777 ;; These utilities and analyzer declaration function are for
778 ;; creating an analyzer which produces new macros in the macro table.
779 ;;
780 ;; There are two analyzers. One for new macros, and one for removing
781 ;; a macro.
782
783 (defun semantic-lex-spp-first-token-arg-list (token)
784 "If TOKEN is a semantic-list, turn it into a an SPP ARG LIST."
785 (when (and (consp token)
786 (symbolp (car token))
787 (eq 'semantic-list (car token)))
788 ;; Convert TOKEN in place.
789 (let ((argsplit (split-string (semantic-lex-token-text token)
790 "[(), ]" t)))
791 (setcar token 'spp-arg-list)
792 (setcar (nthcdr 1 token) argsplit))
793 ))
794
795 (defun semantic-lex-spp-one-token-and-move-for-macro (max)
796 "Lex up one token, and move to end of that token.
797 Don't go past MAX."
798 (let ((ans (semantic-lex (point) max 0 0)))
799 (if (not ans)
800 (progn (goto-char max)
801 nil)
802 (when (> (semantic-lex-token-end (car ans)) max)
803 (let ((bounds (semantic-lex-token-bounds (car ans))))
804 (setcdr bounds max)))
805 (goto-char (semantic-lex-token-end (car ans)))
806 (car ans))
807 ))
808
809 (defun semantic-lex-spp-stream-for-arglist (token)
810 "Lex up the contents of the arglist TOKEN.
811 Parsing starts inside the parens, and ends at the end of TOKEN."
812 (let ((end (semantic-lex-token-end token))
813 (fresh-toks nil)
814 (toks nil))
815 (save-excursion
816
817 (if (stringp (nth 1 token))
818 ;; If the 2nd part of the token is a string, then we have
819 ;; a token specifically extracted from a buffer. Possibly
820 ;; a different buffer. This means we need to do something
821 ;; nice to parse its contents.
822 (let ((txt (semantic-lex-token-text token)))
823 (semantic-lex-spp-lex-text-string
824 (substring txt 1 (1- (length txt)))))
825
826 ;; This part is like the original
827 (goto-char (semantic-lex-token-start token))
828 ;; A cheat for going into the semantic list.
829 (forward-char 1)
830 (setq fresh-toks (semantic-lex-spp-stream-for-macro (1- end)))
831 (dolist (tok fresh-toks)
832 (when (memq (semantic-lex-token-class tok) '(symbol semantic-list))
833 (setq toks (cons tok toks))))
834
835 (nreverse toks)))))
836
837 (defvar semantic-lex-spp-hack-depth 0
838 "Current depth of recursive calls to `semantic-lex-spp-lex-text-string'.")
839
840 (defun semantic-lex-spp-lex-text-string (text)
841 "Lex the text string TEXT using the current buffer's state.
842 Use this to parse text extracted from a macro as if it came from
843 the current buffer. Since the lexer is designed to only work in
844 a buffer, we need to create a new buffer, and populate it with rules
845 and variable state from the current buffer."
846 (let* ((semantic-lex-spp-hack-depth (1+ semantic-lex-spp-hack-depth))
847 (buf (get-buffer-create (format " *SPP parse hack %d*"
848 semantic-lex-spp-hack-depth)))
849 (mode major-mode)
850 (fresh-toks nil)
851 (toks nil)
852 (origbuff (current-buffer))
853 (important-vars '(semantic-lex-spp-macro-symbol-obarray
854 semantic-lex-spp-project-macro-symbol-obarray
855 semantic-lex-spp-dynamic-macro-symbol-obarray
856 semantic-lex-spp-dynamic-macro-symbol-obarray-stack
857 semantic-lex-spp-expanded-macro-stack
858 ))
859 )
860 (save-excursion
861 (set-buffer buf)
862 (erase-buffer)
863 ;; Below is a painful hack to make sure everything is setup correctly.
864 (when (not (eq major-mode mode))
865 (funcall mode)
866 ;; Hack in mode-local
867 (activate-mode-local-bindings)
868 ;; CHEATER! The following 3 lines are from
869 ;; `semantic-new-buffer-fcn', but we don't want to turn
870 ;; on all the other annoying modes for this little task.
871 (setq semantic-new-buffer-fcn-was-run t)
872 (semantic-lex-init)
873 (semantic-clear-toplevel-cache)
874 (remove-hook 'semantic-lex-reset-hooks 'semantic-lex-spp-reset-hook
875 t)
876 )
877
878 ;; Second Cheat: copy key variables reguarding macro state from the
879 ;; the originating buffer we are parsing. We need to do this every time
880 ;; since the state changes.
881 (dolist (V important-vars)
882 (set V (semantic-buffer-local-value V origbuff)))
883 (insert text)
884 (goto-char (point-min))
885
886 (setq fresh-toks (semantic-lex-spp-stream-for-macro (point-max))))
887
888 (dolist (tok fresh-toks)
889 (when (memq (semantic-lex-token-class tok) '(symbol semantic-list))
890 (setq toks (cons tok toks))))
891
892 (nreverse toks)))
893
894 ;;;; FIRST DRAFT
895 ;; This is the fist version of semantic-lex-spp-stream-for-arglist
896 ;; that worked pretty well. It doesn't work if the TOKEN was derived
897 ;; from some other buffer, in which case it can get the wrong answer
898 ;; or throw an error if the token location in the originating buffer is
899 ;; larger than the current buffer.
900 ;;(defun semantic-lex-spp-stream-for-arglist-orig (token)
901 ;; "Lex up the contents of the arglist TOKEN.
902 ;; Parsing starts inside the parens, and ends at the end of TOKEN."
903 ;; (save-excursion
904 ;; (let ((end (semantic-lex-token-end token))
905 ;; (fresh-toks nil)
906 ;; (toks nil))
907 ;; (goto-char (semantic-lex-token-start token))
908 ;; ;; A cheat for going into the semantic list.
909 ;; (forward-char 1)
910 ;; (setq fresh-toks (semantic-lex-spp-stream-for-macro (1- end)))
911 ;; (dolist (tok fresh-toks)
912 ;; (when (memq (semantic-lex-token-class tok) '(symbol semantic-list))
913 ;; (setq toks (cons tok toks))))
914 ;; (nreverse toks))
915 ;; ))
916
917 ;;;; USING SPLIT
918 ;; This doesn't work, because some arguments passed into a macro
919 ;; might contain non-simple symbol words, which this doesn't handle.
920 ;;
921 ;; Thus, you need a full lex to occur.
922 ;; (defun semantic-lex-spp-stream-for-arglist-split (token)
923 ;; "Lex up the contents of the arglist TOKEN.
924 ;; Parsing starts inside the parens, and ends at the end of TOKEN."
925 ;; (let* ((txt (semantic-lex-token-text token))
926 ;; (split (split-string (substring txt 1 (1- (length txt)))
927 ;; "(), " t))
928 ;; ;; Hack for lexing.
929 ;; (semantic-lex-spp-analyzer-push-tokens-for-symbol nil))
930 ;; (dolist (S split)
931 ;; (semantic-lex-spp-analyzer-push-tokens-for-symbol S 0 1))
932 ;; (reverse semantic-lex-spp-analyzer-push-tokens-for-symbol)))
933
934
935 (defun semantic-lex-spp-stream-for-macro (eos)
936 "Lex up a stream of tokens for a #define statement.
937 Parsing starts at the current point location.
938 EOS is the end of the stream to lex for this macro."
939 (let ((stream nil))
940 (while (< (point) eos)
941 (let* ((tok (semantic-lex-spp-one-token-and-move-for-macro eos))
942 (str (when tok
943 (semantic-lex-token-text tok)))
944 )
945 (if str
946 (push (semantic-lex-token (semantic-lex-token-class tok)
947 (semantic-lex-token-start tok)
948 (semantic-lex-token-end tok)
949 str)
950 stream)
951 ;; Nothing to push.
952 nil)))
953 (goto-char eos)
954 ;; Fix the order
955 (nreverse stream)
956 ))
957
958 (defmacro define-lex-spp-macro-declaration-analyzer (name doc regexp tokidx
959 &rest valform)
960 "Define a lexical analyzer for defining new MACROS.
961 NAME is the name of the analyzer.
962 DOC is the documentation for the analyzer.
963 REGEXP is a regular expression for the analyzer to match.
964 See `define-lex-regex-analyzer' for more on regexp.
965 TOKIDX is an index into REGEXP for which a new lexical token
966 of type `spp-macro-def' is to be created.
967 VALFORM are forms that return the value to be saved for this macro, or nil.
968 When implementing a macro, you can use `semantic-lex-spp-stream-for-macro'
969 to convert text into a lexical stream for storage in the macro."
970 (let ((start (make-symbol "start"))
971 (end (make-symbol "end"))
972 (val (make-symbol "val"))
973 (startpnt (make-symbol "startpnt"))
974 (endpnt (make-symbol "endpnt")))
975 `(define-lex-regex-analyzer ,name
976 ,doc
977 ,regexp
978 (let ((,start (match-beginning ,tokidx))
979 (,end (match-end ,tokidx))
980 (,startpnt semantic-lex-end-point)
981 (,val (save-match-data ,@valform))
982 (,endpnt semantic-lex-end-point))
983 (semantic-lex-spp-symbol-set
984 (buffer-substring-no-properties ,start ,end)
985 ,val)
986 (semantic-lex-push-token
987 (semantic-lex-token 'spp-macro-def
988 ,start ,end))
989 ;; Preserve setting of the end point from the calling macro.
990 (when (and (/= ,startpnt ,endpnt)
991 (/= ,endpnt semantic-lex-end-point))
992 (setq semantic-lex-end-point ,endpnt))
993 ))))
994
995 (defmacro define-lex-spp-macro-undeclaration-analyzer (name doc regexp tokidx)
996 "Undefine a lexical analyzer for defining new MACROS.
997 NAME is the name of the analyzer.
998 DOC is the documentation for the analyzer.
999 REGEXP is a regular expression for the analyzer to match.
1000 See `define-lex-regex-analyzer' for more on regexp.
1001 TOKIDX is an index into REGEXP for which a new lexical token
1002 of type `spp-macro-undef' is to be created."
1003 (let ((start (make-symbol "start"))
1004 (end (make-symbol "end")))
1005 `(define-lex-regex-analyzer ,name
1006 ,doc
1007 ,regexp
1008 (let ((,start (match-beginning ,tokidx))
1009 (,end (match-end ,tokidx))
1010 )
1011 (semantic-lex-spp-symbol-remove
1012 (buffer-substring-no-properties ,start ,end))
1013 (semantic-lex-push-token
1014 (semantic-lex-token 'spp-macro-undef
1015 ,start ,end))
1016 ))))
1017
1018 ;;; INCLUDES
1019 ;;
1020 ;; These analyzers help a language define how include files
1021 ;; are identified. These are ONLY for languages that perform
1022 ;; an actual textual includesion, and not for imports.
1023 ;;
1024 ;; This section is supposed to allow the macros from the headers to be
1025 ;; added to the local dynamic macro table, but that hasn't been
1026 ;; written yet.
1027 ;;
1028 (defcustom semantic-lex-spp-use-headers-flag nil
1029 "*Non-nil means to pre-parse headers as we go.
1030 For languages that use the Semantic pre-processor, this can
1031 improve the accuracy of parsed files where include files
1032 can change the state of what's parsed in the current file.
1033
1034 Note: Note implemented yet"
1035 :group 'semantic
1036 :type 'boolean)
1037
1038 (defun semantic-lex-spp-merge-header (name)
1039 "Extract and merge any macros from the header with NAME.
1040 Finds the header file belonging to NAME, gets the macros
1041 from that file, and then merge the macros with our current
1042 symbol table."
1043 (when semantic-lex-spp-use-headers-flag
1044 ;; @todo - do this someday, ok?
1045 ))
1046
1047 (defmacro define-lex-spp-include-analyzer (name doc regexp tokidx
1048 &rest valform)
1049 "Define a lexical analyzer for defining a new INCLUDE lexical token.
1050 Macros defined in the found include will be added to our running table
1051 at the time the include statement is found.
1052 NAME is the name of the analyzer.
1053 DOC is the documentation for the analyzer.
1054 REGEXP is a regular expression for the analyzer to match.
1055 See `define-lex-regex-analyzer' for more on regexp.
1056 TOKIDX is an index into REGEXP for which a new lexical token
1057 of type `spp-macro-include' is to be created.
1058 VALFORM are forms that return the name of the thing being included, and the
1059 type of include. The return value should be of the form:
1060 (NAME . TYPE)
1061 where NAME is the name of the include, and TYPE is the type of the include,
1062 where a valid symbol is 'system, or nil."
1063 (let ((start (make-symbol "start"))
1064 (end (make-symbol "end"))
1065 (val (make-symbol "val"))
1066 (startpnt (make-symbol "startpnt"))
1067 (endpnt (make-symbol "endpnt")))
1068 `(define-lex-regex-analyzer ,name
1069 ,doc
1070 ,regexp
1071 (let ((,start (match-beginning ,tokidx))
1072 (,end (match-end ,tokidx))
1073 (,startpnt semantic-lex-end-point)
1074 (,val (save-match-data ,@valform))
1075 (,endpnt semantic-lex-end-point))
1076 ;;(message "(car ,val) -> %S" (car ,val))
1077 (semantic-lex-spp-merge-header (car ,val))
1078 (semantic-lex-push-token
1079 (semantic-lex-token (if (eq (cdr ,val) 'system)
1080 'spp-system-include
1081 'spp-include)
1082 ,start ,end
1083 (car ,val)))
1084 ;; Preserve setting of the end point from the calling macro.
1085 (when (and (/= ,startpnt ,endpnt)
1086 (/= ,endpnt semantic-lex-end-point))
1087 (setq semantic-lex-end-point ,endpnt))
1088 ))))
1089
1090 ;;; EIEIO USAGE
1091 ;;
1092 ;; Semanticdb can save off macro tables for quick lookup later.
1093 ;;
1094 ;; These routines are for saving macro lists into an EIEIO persistent
1095 ;; file.
1096 (defvar semantic-lex-spp-macro-max-length-to-save 200
1097 "*Maximum length of an SPP macro before we opt to not save it.")
1098
1099 (defun semantic-lex-spp-table-write-slot-value (value)
1100 "Write out the VALUE of a slot for EIEIO.
1101 The VALUE is a spp lexical table."
1102 (if (not value)
1103 (princ "nil")
1104 (princ "\n '(")
1105 ;(princ value)
1106 (dolist (sym value)
1107 (princ "(")
1108 (prin1 (car sym))
1109 (let* ((first (car (cdr sym)))
1110 (rest (cdr sym)))
1111 (when (not (listp first))
1112 (error "Error in macro \"%s\"" (car sym)))
1113 (when (eq (car first) 'spp-arg-list)
1114 (princ " ")
1115 (prin1 first)
1116 (setq rest (cdr rest))
1117 )
1118
1119 (when rest
1120 (princ " . ")
1121 (let ((len (length (cdr rest))))
1122 (cond ((< len 2)
1123 (condition-case nil
1124 (prin1 rest)
1125 (error
1126 (princ "nil ;; Error writing macro\n"))))
1127 ((< len semantic-lex-spp-macro-max-length-to-save)
1128 (princ "\n ")
1129 (condition-case nil
1130 (prin1 rest)
1131 (error
1132 (princ "nil ;; Error writing macro\n ")))
1133 )
1134 (t ;; Too Long!
1135 (princ "nil ;; Too Long!\n ")
1136 ))))
1137 )
1138 (princ ")\n ")
1139 )
1140 (princ ")\n"))
1141 )
1142
1143 ;;; TESTS
1144 ;;
1145 (defun semantic-lex-spp-write-test ()
1146 "Test the semantic tag writer against the current buffer."
1147 (interactive)
1148 (with-output-to-temp-buffer "*SPP Write Test*"
1149 (semantic-lex-spp-table-write-slot-value
1150 (semantic-lex-spp-save-table))))
1151
1152 (defun semantic-lex-spp-write-utest ()
1153 "Unit test using the test spp file to test the slot write fcn."
1154 (interactive)
1155 (let* ((sem (locate-library "semantic-lex-spp.el"))
1156 (dir (file-name-directory sem)))
1157 (save-excursion
1158 (set-buffer (find-file-noselect
1159 (expand-file-name "tests/testsppreplace.c"
1160 dir)))
1161 (semantic-lex-spp-write-test))))
1162
1163 ;;; MACRO TABLE DEBUG
1164 ;;
1165 (defun semantic-lex-spp-describe (&optional buffer)
1166 "Describe the current list of spp macros for BUFFER.
1167 If BUFFER is not provided, use the current buffer."
1168 (interactive)
1169 (let ((syms (save-excursion
1170 (if buffer (set-buffer buffer))
1171 (semantic-lex-spp-macros)))
1172 (sym nil))
1173 (with-output-to-temp-buffer "*SPP MACROS*"
1174 (princ "Macro\t\tValue\n")
1175 (while syms
1176 (setq sym (car syms)
1177 syms (cdr syms))
1178 (princ (symbol-name sym))
1179 (princ "\t")
1180 (if (< (length (symbol-name sym)) 8)
1181 (princ "\t"))
1182 (prin1 (symbol-value sym))
1183 (princ "\n")
1184 ))))
1185
1186 ;;; EDEBUG Handlers
1187 ;;
1188 (add-hook
1189 'edebug-setup-hook
1190 #'(lambda ()
1191
1192 (def-edebug-spec define-lex-spp-macro-declaration-analyzer
1193 (&define name stringp stringp form def-body)
1194 )
1195
1196 (def-edebug-spec define-lex-spp-macro-undeclaration-analyzer
1197 (&define name stringp stringp form)
1198 )
1199
1200 (def-edebug-spec define-lex-spp-include-analyzer
1201 (&define name stringp stringp form def-body)
1202 )
1203 ))
1204
1205
1206 (provide 'semantic/lex-spp)
1207
1208 ;;; semantic-lex-spp.el ends here