libguile/c-tokenize.lex

   1 %option noyywrap
   2 %option nounput
   3 %pointer
   4
   5 EOL             \n
   6 SPACE           [ \t\v\f]
   7 WS              [ \t\v\n\f]
   8 DIGIT           [0-9]
   9 LETTER          [a-zA-Z_]
  10 OCTDIGIT        [0-7]
  11 HEXDIGIT        [a-fA-F0-9]
  12 EXPONENT        [Ee][+-]?{DIGIT}+
  13 FLOQUAL         (f|F|l|L)
  14 INTQUAL         (l|L|ll|LL|lL|Ll|u|U)
  15
  16 %{
  17
  18 #include <stdio.h>
  19 #include <stdlib.h>
  20 #include <string.h>
  21
  22 int yylex(void);
  23
  24 int filter_snarfage = 0;
  25 int print = 1;
  26
  27 enum t_state {
  28     SKIP,
  29     MULTILINE,
  30     MULTILINE_COOKIE,
  31     COOKIE,
  32 };
  33
  34 enum t_state state = SKIP;
  35 int cookie_was_last = 0;
  36
  37 #define OUT_RAW(type,text) if (print) printf ("(%s . \"%s\")\n", #type, text)
  38
  39 #define OUT_T(type)        OUT_RAW (type, yytext)
  40 #define OUT_S              if (print) printf ("%s\n", yytext)
  41 #define OUT(type)          if (print) printf ("%s\n", #type)
  42
  43 #define IS_COOKIE cookie_was_last = 1
  44 #define IS_NOT_COOKIE cookie_was_last = 0
  45
  46 %}
  47
  48 %%
  49
  50 \/\*(\n|[^*]|\*[^/])*\*\/                  { OUT_T (comment); }
  51
  52 ({SPACE}*(\\\n)*{SPACE}*)+                 ;
  53
  54 ({SPACE}*\n*{SPACE}*)+                     { OUT(eol); }
  55
  56 #                                          { OUT(hash); IS_NOT_COOKIE; }
  57
  58 {LETTER}({LETTER}|{DIGIT})*                { OUT_T (id); IS_NOT_COOKIE; }
  59
  60 0[xX]{HEXDIGIT}+{INTQUAL}?                 { OUT_RAW (int_hex, yytext + 2); IS_NOT_COOKIE; }
  61 0{OCTDIGIT}+{INTQUAL}?                     { OUT_RAW (int_oct, yytext + 1); IS_NOT_COOKIE; }
  62 {DIGIT}+{INTQUAL}?                         { OUT_T (int_dec); IS_NOT_COOKIE; }
  63
  64 L?\'(\\.|[^\\\'])+\'                       { OUT_T (char); IS_NOT_COOKIE; }
  65
  66 {DIGIT}+{EXPONENT}{FLOQUAL}?               { OUT_T (flo_dec); IS_NOT_COOKIE; }
  67 {DIGIT}*"."{DIGIT}+({EXPONENT})?{FLOQUAL}? { OUT_T (flo_dec); IS_NOT_COOKIE; }
  68 {DIGIT}+"."{DIGIT}*({EXPONENT})?{FLOQUAL}? { OUT_T (flo_dec); IS_NOT_COOKIE; }
  69
  70 L?\"(\\.|[^\\\"])*\"                       { OUT_S; IS_NOT_COOKIE; }
  71
  72 "..."                                      { OUT (ellipsis); IS_NOT_COOKIE; }
  73
  74 ">>="                                      { OUT (shift_right_assign); IS_NOT_COOKIE; }
  75 "<<="                                      { OUT (shift_left_assign); IS_NOT_COOKIE; }
  76 "+="                                       { OUT (add_assign); IS_NOT_COOKIE; }
  77 "-="                                       { OUT (sub_assign); IS_NOT_COOKIE; }
  78 "*="                                       { OUT (mul-assign); IS_NOT_COOKIE; }
  79 "/="                                       { OUT (div_assign); IS_NOT_COOKIE; }
  80 "%="                                       { OUT (mod_assign); IS_NOT_COOKIE; }
  81 "&="                                       { OUT (logand_assign); IS_NOT_COOKIE; }
  82 "^="                                       { OUT (logxor_assign); IS_NOT_COOKIE; }
  83 "|="                                       { OUT (logior_assign); IS_NOT_COOKIE; }
  84 ">>"                                       { OUT (right_shift); IS_NOT_COOKIE; }
  85 "<<"                                       { OUT (left_shift); IS_NOT_COOKIE; }
  86 "++"                                       { OUT (inc); IS_NOT_COOKIE; }
  87 "--"                                       { OUT (dec); IS_NOT_COOKIE; }
  88 "->"                                       { OUT (ptr); IS_NOT_COOKIE; }
  89 "&&"                                       { OUT (and); IS_NOT_COOKIE; }
  90 "||"                                       { OUT (or); IS_NOT_COOKIE; }
  91 "<="                                       { OUT (le); IS_NOT_COOKIE; }
  92 ">="                                       { OUT (ge); IS_NOT_COOKIE; }
  93 "=="                                       { OUT (eq); IS_NOT_COOKIE; }
  94 "!="                                       { OUT (ne); IS_NOT_COOKIE; }
  95 ";"                                        { OUT (semicolon); IS_NOT_COOKIE; }
  96
  97 ("{"|"<%")                                 {
  98   OUT (brace_open);
  99   if (filter_snarfage && cookie_was_last && state == COOKIE)
 100     state = MULTILINE;
 101   IS_NOT_COOKIE; }
 102
 103 ("}"|"%>")                                 {
 104   OUT (brace_close);
 105   if (filter_snarfage && cookie_was_last && state == MULTILINE_COOKIE) {
 106     state = SKIP;
 107     print = 0;
 108   }
 109   IS_NOT_COOKIE; }
 110
 111 ","                                        { OUT (comma); IS_NOT_COOKIE; }
 112 ":"                                        { OUT (colon); IS_NOT_COOKIE; }
 113 "="                                        { OUT (assign); IS_NOT_COOKIE; }
 114 "("                                        { OUT (paren_open); IS_NOT_COOKIE; }
 115 ")"                                        { OUT (paren_close); IS_NOT_COOKIE; }
 116 ("["|"<:")                                 { OUT (bracket_open); IS_NOT_COOKIE; }
 117 ("]"|":>")                                 { OUT (bracket_close); IS_NOT_COOKIE; }
 118 "."                                        { OUT (dot); IS_NOT_COOKIE; }
 119 "&"                                        { OUT (amp); IS_NOT_COOKIE; }
 120 "!"                                        { OUT (bang); IS_NOT_COOKIE; }
 121 "~"                                        { OUT (tilde); IS_NOT_COOKIE; }
 122 "-"                                        { OUT (minus); IS_NOT_COOKIE; }
 123 "+"                                        { OUT (plus); IS_NOT_COOKIE; }
 124 "*"                                        { OUT (star); IS_NOT_COOKIE; }
 125 "/"                                        { OUT (slash); IS_NOT_COOKIE; }
 126 "%"                                        { OUT (percent); IS_NOT_COOKIE; }
 127 "<"                                        { OUT (lt); IS_NOT_COOKIE; }
 128 ">"                                        { OUT (gt); IS_NOT_COOKIE; }
 129
 130 \^{WS}*\^                                  {
 131   if (filter_snarfage)
 132     switch (state) {
 133     case SKIP:
 134       state = COOKIE;
 135       print = 1;
 136       OUT (snarf_cookie);
 137       break;
 138     case MULTILINE:
 139     case MULTILINE_COOKIE:
 140       state = MULTILINE_COOKIE;
 141       OUT (snarf_cookie);
 142       break;
 143     case COOKIE:
 144       state = SKIP;
 145       OUT (snarf_cookie);
 146       print = 0;
 147       break;
 148     default:
 149       /* whoops */
 150       abort ();
 151       break;
 152     }
 153   else
 154     OUT (snarf_cookie);
 155
 156   IS_COOKIE; }
 157
 158 "^"                                        { OUT (caret); IS_NOT_COOKIE; }
 159 "|"                                        { OUT (pipe); IS_NOT_COOKIE; }
 160 "?"                                        { OUT (question); IS_NOT_COOKIE; }
 161
 162 .                                          { fprintf (stderr, "*%s", yytext);  fflush (stderr); IS_NOT_COOKIE; }
 163
 164 %%
 165
 166 int
 167 main (int argc, char *argv[])
 168 {
 169     if (argc > 1 && !strcmp (argv[1], "--filter-snarfage")) {
 170         filter_snarfage = 1;
 171         print = 0;
 172     }
 173
 174     yylex ();
 175
 176     return EXIT_SUCCESS;
 177 }
 178
 179 /*
 180   Local Variables:
 181   c-file-style: "gnu"
 182   End:
 183 */