[bpt/guile.git] / libguile / c-tokenize.lex

%option noyywrap
%option nounput
%pointer

EOL             \n
SPACE           [ \t\v\f]
WS              [ \t\v\n\f]
DIGIT		[0-9]
LETTER		[a-zA-Z_]
OCTDIGIT	[0-7]
HEXDIGIT	[a-fA-F0-9]
EXPONENT	[Ee][+-]?{DIGIT}+
FLOQUAL		(f|F|l|L)
INTQUAL		(l|L|ll|LL|lL|Ll|u|U)

%{
    
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

/* Prevent compilation of static input() function in generated scanner
   code.  This function is never actually used, and GCC 4.3 will emit
   an error for that. */
#define YY_NO_INPUT

int yylex(void);

int yyget_lineno (void);
FILE *yyget_in (void);
FILE *yyget_out (void);
int yyget_leng (void);
char *yyget_text (void);
void yyset_lineno (int line_number);
void yyset_in (FILE * in_str);
void yyset_out (FILE * out_str);
int yyget_debug (void);
void yyset_debug (int  bdebug);
int yylex_destroy (void);
 
int filter_snarfage = 0;
int print = 1; 

enum t_state {
    SKIP,
    MULTILINE,
    MULTILINE_COOKIE,
    COOKIE
};

enum t_state state = SKIP;
int cookie_was_last = 0; 

#define OUT_RAW(type,text) if (print) printf ("(%s . \"%s\")\n", #type, text)
 
#define OUT_T(type)        OUT_RAW (type, yytext)
#define OUT_S              if (print) printf ("%s\n", yytext)
#define OUT(type)          if (print) printf ("%s\n", #type)

#define IS_COOKIE cookie_was_last = 1
#define IS_NOT_COOKIE cookie_was_last = 0
 
%}

%%

\/\*(\n|[^*]|\*[^/])*\*\/                  { OUT_T (comment); }

({SPACE}*(\\\n)*{SPACE}*)+                 ;

({SPACE}*\n*{SPACE}*)+                     { OUT(eol); }

#.*\n                                      { OUT(hash); IS_NOT_COOKIE; }

{LETTER}({LETTER}|{DIGIT})*                { OUT_T (id); IS_NOT_COOKIE; }

0[xX]{HEXDIGIT}+{INTQUAL}?                 { OUT_RAW (int_hex, yytext + 2); IS_NOT_COOKIE; }
0{OCTDIGIT}+{INTQUAL}?                     { OUT_RAW (int_oct, yytext + 1); IS_NOT_COOKIE; }
{DIGIT}+{INTQUAL}?                         { OUT_T (int_dec); IS_NOT_COOKIE; }

L?\'(\\.|[^\\\'])+\'                       { OUT_T (char); IS_NOT_COOKIE; }

{DIGIT}+{EXPONENT}{FLOQUAL}?               { OUT_T (flo_dec); IS_NOT_COOKIE; }
{DIGIT}*"."{DIGIT}+({EXPONENT})?{FLOQUAL}? { OUT_T (flo_dec); IS_NOT_COOKIE; }
{DIGIT}+"."{DIGIT}*({EXPONENT})?{FLOQUAL}? { OUT_T (flo_dec); IS_NOT_COOKIE; }

L?\"(\\.|[^\\\"])*\"                       { OUT_S; IS_NOT_COOKIE; }

"..."                                      { OUT (ellipsis); IS_NOT_COOKIE; }

">>="                                      { OUT (shift_right_assign); IS_NOT_COOKIE; }
"<<="                                      { OUT (shift_left_assign); IS_NOT_COOKIE; }
"+="                                       { OUT (add_assign); IS_NOT_COOKIE; }
"-="                                       { OUT (sub_assign); IS_NOT_COOKIE; }
"*="                                       { OUT (mul-assign); IS_NOT_COOKIE; }
"/="                                       { OUT (div_assign); IS_NOT_COOKIE; }
"%="                                       { OUT (mod_assign); IS_NOT_COOKIE; }
"&="                                       { OUT (logand_assign); IS_NOT_COOKIE; }
"^="                                       { OUT (logxor_assign); IS_NOT_COOKIE; }
"|="                                       { OUT (logior_assign); IS_NOT_COOKIE; }
">>"                                       { OUT (right_shift); IS_NOT_COOKIE; }
"<<"                                       { OUT (left_shift); IS_NOT_COOKIE; }
"++"                                       { OUT (inc); IS_NOT_COOKIE; }
"--"                                       { OUT (dec); IS_NOT_COOKIE; }
"->"                                       { OUT (ptr); IS_NOT_COOKIE; }
"&&"                                       { OUT (and); IS_NOT_COOKIE; }
"||"                                       { OUT (or); IS_NOT_COOKIE; }
"<="                                       { OUT (le); IS_NOT_COOKIE; }
">="                                       { OUT (ge); IS_NOT_COOKIE; }
"=="                                       { OUT (eq); IS_NOT_COOKIE; }
"!="                                       { OUT (ne); IS_NOT_COOKIE; }
";"                                        { OUT (semicolon); IS_NOT_COOKIE; }

("{"|"<%")                                 {
  OUT (brace_open);
  if (filter_snarfage && cookie_was_last && state == COOKIE)
    state = MULTILINE;
  IS_NOT_COOKIE; }

("}"|"%>")                                 {
  OUT (brace_close);
  if (filter_snarfage && cookie_was_last && state == MULTILINE_COOKIE) {
    state = SKIP;
    print = 0;
  }
  IS_NOT_COOKIE; }

","                                        { OUT (comma); IS_NOT_COOKIE; }
":"                                        { OUT (colon); IS_NOT_COOKIE; }
"="                                        { OUT (assign); IS_NOT_COOKIE; }
"("                                        { OUT (paren_open); IS_NOT_COOKIE; }
")"                                        { OUT (paren_close); IS_NOT_COOKIE; }
("["|"<:")                                 { OUT (bracket_open); IS_NOT_COOKIE; }
("]"|":>")                                 { OUT (bracket_close); IS_NOT_COOKIE; }
"."                                        { OUT (dot); IS_NOT_COOKIE; }
"&"                                        { OUT (amp); IS_NOT_COOKIE; }
"!"                                        { OUT (bang); IS_NOT_COOKIE; }
"~"                                        { OUT (tilde); IS_NOT_COOKIE; }
"-"                                        { OUT (minus); IS_NOT_COOKIE; }
"+"                                        { OUT (plus); IS_NOT_COOKIE; }
"*"                                        { OUT (star); IS_NOT_COOKIE; }
"/"                                        { OUT (slash); IS_NOT_COOKIE; }
"%"                                        { OUT (percent); IS_NOT_COOKIE; }
"<"                                        { OUT (lt); IS_NOT_COOKIE; }
">"                                        { OUT (gt); IS_NOT_COOKIE; }

\^{WS}*\^                                  {
  if (filter_snarfage)
    switch (state) {
    case SKIP:
      state = COOKIE;
      print = 1;
      OUT (snarf_cookie);
      break;
    case MULTILINE:
    case MULTILINE_COOKIE:
      state = MULTILINE_COOKIE;
      OUT (snarf_cookie);
      break;
    case COOKIE:
      state = SKIP;
      OUT (snarf_cookie);
      print = 0;
      break;
    default:
      /* whoops */
      abort ();
      break;
    }
  else
    OUT (snarf_cookie);
  
  IS_COOKIE; }

"^"                                        { OUT (caret); IS_NOT_COOKIE; }
"|"                                        { OUT (pipe); IS_NOT_COOKIE; }
"?"                                        { OUT (question); IS_NOT_COOKIE; }

.                                          { fprintf (stderr, "*%s", yytext);  fflush (stderr); IS_NOT_COOKIE; }

%%

int
main (int argc, char *argv[])
{
    if (argc > 1 && !strcmp (argv[1], "--filter-snarfage")) {
        filter_snarfage = 1;
        print = 0;
    }

    yylex ();

    return EXIT_SUCCESS;
}

/*
  Local Variables:
  c-file-style: "gnu"
  End:
*/
Commit	Line	Data
ac13d9d2	1	%option noyywrap
e80bea70	2	%option nounput
ac13d9d2 ML	3	%pointer
	4
	5	EOL \n
	6	SPACE [ \t\v\f]
	7	WS [ \t\v\n\f]
	8	DIGIT [0-9]
	9	LETTER [a-zA-Z_]
	10	OCTDIGIT [0-7]
	11	HEXDIGIT [a-fA-F0-9]
	12	EXPONENT [Ee][+-]?{DIGIT}+
	13	FLOQUAL (f\|F\|l\|L)
	14	INTQUAL (l\|L\|ll\|LL\|lL\|Ll\|u\|U)
	15
	16	%{
	17
	18	#include <stdio.h>
	19	#include <stdlib.h>
	20	#include <string.h>
6d4e1f62 NJ	21
	22	/* Prevent compilation of static input() function in generated scanner
	23	code. This function is never actually used, and GCC 4.3 will emit
	24	an error for that. */
	25	#define YY_NO_INPUT
	26
7e59d9d4 MD	27	int yylex(void);
	28
	29	int yyget_lineno (void);
	30	FILE *yyget_in (void);
	31	FILE *yyget_out (void);
	32	int yyget_leng (void);
	33	char *yyget_text (void);
	34	void yyset_lineno (int line_number);
	35	void yyset_in (FILE * in_str);
	36	void yyset_out (FILE * out_str);
	37	int yyget_debug (void);
	38	void yyset_debug (int bdebug);
	39	int yylex_destroy (void);
	40
ac13d9d2 ML	41	int filter_snarfage = 0;
	42	int print = 1;
	43
	44	enum t_state {
	45	SKIP,
	46	MULTILINE,
	47	MULTILINE_COOKIE,
62024674	48	COOKIE
ac13d9d2 ML	49	};
	50
	51	enum t_state state = SKIP;
	52	int cookie_was_last = 0;
	53
	54	#define OUT_RAW(type,text) if (print) printf ("(%s . \"%s\")\n", #type, text)
	55
	56	#define OUT_T(type) OUT_RAW (type, yytext)
	57	#define OUT_S if (print) printf ("%s\n", yytext)
	58	#define OUT(type) if (print) printf ("%s\n", #type)
	59
	60	#define IS_COOKIE cookie_was_last = 1
	61	#define IS_NOT_COOKIE cookie_was_last = 0
	62
	63	%}
	64
	65	%%
	66
	67	\/\(\n\|[^]\|\[^/])\*\/ { OUT_T (comment); }
	68
	69	({SPACE}(\\\n){SPACE}*)+ ;
	70
	71	({SPACE}\n{SPACE}*)+ { OUT(eol); }
	72
6d7c6cf4	73	#.*\n { OUT(hash); IS_NOT_COOKIE; }
ac13d9d2 ML	74
	75	{LETTER}({LETTER}\|{DIGIT})* { OUT_T (id); IS_NOT_COOKIE; }
	76
	77	0[xX]{HEXDIGIT}+{INTQUAL}? { OUT_RAW (int_hex, yytext + 2); IS_NOT_COOKIE; }
	78	0{OCTDIGIT}+{INTQUAL}? { OUT_RAW (int_oct, yytext + 1); IS_NOT_COOKIE; }
	79	{DIGIT}+{INTQUAL}? { OUT_T (int_dec); IS_NOT_COOKIE; }
	80
	81	L?\'(\\.\|[^\\\'])+\' { OUT_T (char); IS_NOT_COOKIE; }
	82
	83	{DIGIT}+{EXPONENT}{FLOQUAL}? { OUT_T (flo_dec); IS_NOT_COOKIE; }
	84	{DIGIT}*"."{DIGIT}+({EXPONENT})?{FLOQUAL}? { OUT_T (flo_dec); IS_NOT_COOKIE; }
	85	{DIGIT}+"."{DIGIT}*({EXPONENT})?{FLOQUAL}? { OUT_T (flo_dec); IS_NOT_COOKIE; }
	86
	87	L?\"(\\.\|[^\\\"])*\" { OUT_S; IS_NOT_COOKIE; }
	88
	89	"..." { OUT (ellipsis); IS_NOT_COOKIE; }
	90
	91	">>=" { OUT (shift_right_assign); IS_NOT_COOKIE; }
	92	"<<=" { OUT (shift_left_assign); IS_NOT_COOKIE; }
	93	"+=" { OUT (add_assign); IS_NOT_COOKIE; }
	94	"-=" { OUT (sub_assign); IS_NOT_COOKIE; }
	95	"*=" { OUT (mul-assign); IS_NOT_COOKIE; }
	96	"/=" { OUT (div_assign); IS_NOT_COOKIE; }
	97	"%=" { OUT (mod_assign); IS_NOT_COOKIE; }
	98	"&=" { OUT (logand_assign); IS_NOT_COOKIE; }
	99	"^=" { OUT (logxor_assign); IS_NOT_COOKIE; }
	100	"\|=" { OUT (logior_assign); IS_NOT_COOKIE; }
	101	">>" { OUT (right_shift); IS_NOT_COOKIE; }
	102	"<<" { OUT (left_shift); IS_NOT_COOKIE; }
	103	"++" { OUT (inc); IS_NOT_COOKIE; }
	104	"--" { OUT (dec); IS_NOT_COOKIE; }
	105	"->" { OUT (ptr); IS_NOT_COOKIE; }
	106	"&&" { OUT (and); IS_NOT_COOKIE; }
	107	"\|\|" { OUT (or); IS_NOT_COOKIE; }
	108	"<=" { OUT (le); IS_NOT_COOKIE; }
	109	">=" { OUT (ge); IS_NOT_COOKIE; }
	110	"==" { OUT (eq); IS_NOT_COOKIE; }
	111	"!=" { OUT (ne); IS_NOT_COOKIE; }
	112	";" { OUT (semicolon); IS_NOT_COOKIE; }
	113
	114	("{"\|"<%") {
	115	OUT (brace_open);
	116	if (filter_snarfage && cookie_was_last && state == COOKIE)
	117	state = MULTILINE;
	118	IS_NOT_COOKIE; }
	119
	120	("}"\|"%>") {
	121	OUT (brace_close);
	122	if (filter_snarfage && cookie_was_last && state == MULTILINE_COOKIE) {
	123	state = SKIP;
	124	print = 0;
	125	}
	126	IS_NOT_COOKIE; }
	127
	128	"," { OUT (comma); IS_NOT_COOKIE; }
	129	":" { OUT (colon); IS_NOT_COOKIE; }
	130	"=" { OUT (assign); IS_NOT_COOKIE; }
	131	"(" { OUT (paren_open); IS_NOT_COOKIE; }
	132	")" { OUT (paren_close); IS_NOT_COOKIE; }
	133	("["\|"<:") { OUT (bracket_open); IS_NOT_COOKIE; }
	134	("]"\|":>") { OUT (bracket_close); IS_NOT_COOKIE; }
	135	"." { OUT (dot); IS_NOT_COOKIE; }
	136	"&" { OUT (amp); IS_NOT_COOKIE; }
	137	"!" { OUT (bang); IS_NOT_COOKIE; }
138	"~" { OUT (tilde); IS_NOT_COOKIE; }
139	"-" { OUT (minus); IS_NOT_COOKIE; }
140	"+" { OUT (plus); IS_NOT_COOKIE; }
141	"*" { OUT (star); IS_NOT_COOKIE; }
142	"/" { OUT (slash); IS_NOT_COOKIE; }
143	"%" { OUT (percent); IS_NOT_COOKIE; }
144	"<" { OUT (lt); IS_NOT_COOKIE; }
145	">" { OUT (gt); IS_NOT_COOKIE; }
146
147	\^{WS}*\^ {
148	if (filter_snarfage)
149	switch (state) {
150	case SKIP:
151	state = COOKIE;
152	print = 1;
153	OUT (snarf_cookie);
154	break;
155	case MULTILINE:
156	case MULTILINE_COOKIE:
157	state = MULTILINE_COOKIE;
158	OUT (snarf_cookie);
159	break;
160	case COOKIE:
161	state = SKIP;
162	OUT (snarf_cookie);
163	print = 0;
164	break;
165	default:
166	/* whoops */
167	abort ();
168	break;
169	}
170	else
171	OUT (snarf_cookie);
172
173	IS_COOKIE; }
174
175	"^" { OUT (caret); IS_NOT_COOKIE; }
176	"\|" { OUT (pipe); IS_NOT_COOKIE; }
177	"?" { OUT (question); IS_NOT_COOKIE; }
178
179	. { fprintf (stderr, "*%s", yytext); fflush (stderr); IS_NOT_COOKIE; }
180
181	%%
182
183	int
184	main (int argc, char *argv[])
185	{
186	if (argc > 1 && !strcmp (argv[1], "--filter-snarfage")) {
187	filter_snarfage = 1;
188	print = 0;
189	}
190
191	yylex ();
192
193	return EXIT_SUCCESS;
194	}
195
196	/*
197	Local Variables:
198	c-file-style: "gnu"
199	End:
200	*/