libguile/c-tokenize.lex

   1 %option noyywrap
   2 %pointer
   3
   4 EOL             \n
   5 SPACE           [ \t\v\f]
   6 WS              [ \t\v\n\f]
   7 DIGIT           [0-9]
   8 LETTER          [a-zA-Z_]
   9 OCTDIGIT        [0-7]
  10 HEXDIGIT        [a-fA-F0-9]
  11 EXPONENT        [Ee][+-]?{DIGIT}+
  12 FLOQUAL         (f|F|l|L)
  13 INTQUAL         (l|L|ll|LL|lL|Ll|u|U)
  14
  15 %{
  16
  17 #include <stdio.h>
  18 #include <stdlib.h>
  19 #include <string.h>
  20
  21 int filter_snarfage = 0;
  22 int print = 1;
  23
  24 enum t_state {
  25     SKIP,
  26     MULTILINE,
  27     MULTILINE_COOKIE,
  28     COOKIE,
  29 };
  30
  31 enum t_state state = SKIP;
  32 int cookie_was_last = 0;
  33
  34 #define OUT_RAW(type,text) if (print) printf ("(%s . \"%s\")\n", #type, text)
  35
  36 #define OUT_T(type)        OUT_RAW (type, yytext)
  37 #define OUT_S              if (print) printf ("%s\n", yytext)
  38 #define OUT(type)          if (print) printf ("%s\n", #type)
  39
  40 #define IS_COOKIE cookie_was_last = 1
  41 #define IS_NOT_COOKIE cookie_was_last = 0
  42
  43 %}
  44
  45 %%
  46
  47 \/\*(\n|[^*]|\*[^/])*\*\/                  { OUT_T (comment); }
  48
  49 ({SPACE}*(\\\n)*{SPACE}*)+                 ;
  50
  51 ({SPACE}*\n*{SPACE}*)+                     { OUT(eol); }
  52
  53 #                                          { OUT(hash); IS_NOT_COOKIE; }
  54
  55 {LETTER}({LETTER}|{DIGIT})*                { OUT_T (id); IS_NOT_COOKIE; }
  56
  57 0[xX]{HEXDIGIT}+{INTQUAL}?                 { OUT_RAW (int_hex, yytext + 2); IS_NOT_COOKIE; }
  58 0{OCTDIGIT}+{INTQUAL}?                     { OUT_RAW (int_oct, yytext + 1); IS_NOT_COOKIE; }
  59 {DIGIT}+{INTQUAL}?                         { OUT_T (int_dec); IS_NOT_COOKIE; }
  60
  61 L?\'(\\.|[^\\\'])+\'                       { OUT_T (char); IS_NOT_COOKIE; }
  62
  63 {DIGIT}+{EXPONENT}{FLOQUAL}?               { OUT_T (flo_dec); IS_NOT_COOKIE; }
  64 {DIGIT}*"."{DIGIT}+({EXPONENT})?{FLOQUAL}? { OUT_T (flo_dec); IS_NOT_COOKIE; }
  65 {DIGIT}+"."{DIGIT}*({EXPONENT})?{FLOQUAL}? { OUT_T (flo_dec); IS_NOT_COOKIE; }
  66
  67 L?\"(\\.|[^\\\"])*\"                       { OUT_S; IS_NOT_COOKIE; }
  68
  69 "..."                                      { OUT (ellipsis); IS_NOT_COOKIE; }
  70
  71 ">>="                                      { OUT (shift_right_assign); IS_NOT_COOKIE; }
  72 "<<="                                      { OUT (shift_left_assign); IS_NOT_COOKIE; }
  73 "+="                                       { OUT (add_assign); IS_NOT_COOKIE; }
  74 "-="                                       { OUT (sub_assign); IS_NOT_COOKIE; }
  75 "*="                                       { OUT (mul-assign); IS_NOT_COOKIE; }
  76 "/="                                       { OUT (div_assign); IS_NOT_COOKIE; }
  77 "%="                                       { OUT (mod_assign); IS_NOT_COOKIE; }
  78 "&="                                       { OUT (logand_assign); IS_NOT_COOKIE; }
  79 "^="                                       { OUT (logxor_assign); IS_NOT_COOKIE; }
  80 "|="                                       { OUT (logior_assign); IS_NOT_COOKIE; }
  81 ">>"                                       { OUT (right_shift); IS_NOT_COOKIE; }
  82 "<<"                                       { OUT (left_shift); IS_NOT_COOKIE; }
  83 "++"                                       { OUT (inc); IS_NOT_COOKIE; }
  84 "--"                                       { OUT (dec); IS_NOT_COOKIE; }
  85 "->"                                       { OUT (ptr); IS_NOT_COOKIE; }
  86 "&&"                                       { OUT (and); IS_NOT_COOKIE; }
  87 "||"                                       { OUT (or); IS_NOT_COOKIE; }
  88 "<="                                       { OUT (le); IS_NOT_COOKIE; }
  89 ">="                                       { OUT (ge); IS_NOT_COOKIE; }
  90 "=="                                       { OUT (eq); IS_NOT_COOKIE; }
  91 "!="                                       { OUT (ne); IS_NOT_COOKIE; }
  92 ";"                                        { OUT (semicolon); IS_NOT_COOKIE; }
  93
  94 ("{"|"<%")                                 {
  95   OUT (brace_open);
  96   if (filter_snarfage && cookie_was_last && state == COOKIE)
  97     state = MULTILINE;
  98   IS_NOT_COOKIE; }
  99
 100 ("}"|"%>")                                 {
 101   OUT (brace_close);
 102   if (filter_snarfage && cookie_was_last && state == MULTILINE_COOKIE) {
 103     state = SKIP;
 104     print = 0;
 105   }
 106   IS_NOT_COOKIE; }
 107
 108 ","                                        { OUT (comma); IS_NOT_COOKIE; }
 109 ":"                                        { OUT (colon); IS_NOT_COOKIE; }
 110 "="                                        { OUT (assign); IS_NOT_COOKIE; }
 111 "("                                        { OUT (paren_open); IS_NOT_COOKIE; }
 112 ")"                                        { OUT (paren_close); IS_NOT_COOKIE; }
 113 ("["|"<:")                                 { OUT (bracket_open); IS_NOT_COOKIE; }
 114 ("]"|":>")                                 { OUT (bracket_close); IS_NOT_COOKIE; }
 115 "."                                        { OUT (dot); IS_NOT_COOKIE; }
 116 "&"                                        { OUT (amp); IS_NOT_COOKIE; }
 117 "!"                                        { OUT (bang); IS_NOT_COOKIE; }
 118 "~"                                        { OUT (tilde); IS_NOT_COOKIE; }
 119 "-"                                        { OUT (minus); IS_NOT_COOKIE; }
 120 "+"                                        { OUT (plus); IS_NOT_COOKIE; }
 121 "*"                                        { OUT (star); IS_NOT_COOKIE; }
 122 "/"                                        { OUT (slash); IS_NOT_COOKIE; }
 123 "%"                                        { OUT (percent); IS_NOT_COOKIE; }
 124 "<"                                        { OUT (lt); IS_NOT_COOKIE; }
 125 ">"                                        { OUT (gt); IS_NOT_COOKIE; }
 126
 127 \^{WS}*\^                                  {
 128   if (filter_snarfage)
 129     switch (state) {
 130     case SKIP:
 131       state = COOKIE;
 132       print = 1;
 133       OUT (snarf_cookie);
 134       break;
 135     case MULTILINE:
 136     case MULTILINE_COOKIE:
 137       state = MULTILINE_COOKIE;
 138       OUT (snarf_cookie);
 139       break;
 140     case COOKIE:
 141       state = SKIP;
 142       OUT (snarf_cookie);
 143       print = 0;
 144       break;
 145     default:
 146       /* whoops */
 147       abort ();
 148       break;
 149     }
 150   else
 151     OUT (snarf_cookie);
 152
 153   IS_COOKIE; }
 154
 155 "^"                                        { OUT (caret); IS_NOT_COOKIE; }
 156 "|"                                        { OUT (pipe); IS_NOT_COOKIE; }
 157 "?"                                        { OUT (question); IS_NOT_COOKIE; }
 158
 159 .                                          { fprintf (stderr, "*%s", yytext);  fflush (stderr); IS_NOT_COOKIE; }
 160
 161 %%
 162
 163 int
 164 main (int argc, char *argv[])
 165 {
 166     if (argc > 1 && !strcmp (argv[1], "--filter-snarfage")) {
 167         filter_snarfage = 1;
 168         print = 0;
 169     }
 170
 171     yylex ();
 172
 173     return EXIT_SUCCESS;
 174 }
 175
 176 /*
 177   Local Variables:
 178   c-file-style: "gnu"
 179   End:
 180 */