parsing_c/token_c.ml

   1 (* Yoann Padioleau
   2  *
   3  * Copyright (C) 2009 University of Urbana Champaign
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License (GPL)
   7  * version 2 as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * file license.txt for more details.
  13  *)
  14
  15
  16 open Common
  17
  18 (*****************************************************************************)
  19 (* Prelude *)
  20 (*****************************************************************************)
  21
  22 (* This file may seems redundant with the tokens generated by Yacc
  23  * from parser.mly in parser_c.mli. The problem is that we need for
  24  * many reasons to remember in the ast_c the tokens invoved in this
  25  * ast, not just the string, especially for the comment and cpp_passed
  26  * tokens which pour le coup were not in the ast at all. So,
  27  * to avoid recursive mutual dependencies, we provide this file
  28  * so that ast_c does not need to depend on yacc which depends on
  29  * ast_c, etc.
  30  *
  31  * Also, ocamlyacc imposes some stupid constraints on the way we can define
  32  * the token type. ocamlyacc forces us to do a token type that
  33  * cant be a pair of a sum type, it must be directly a sum type.
  34  * We don't have this constraint here.
  35  *
  36  * Also, some yacc tokens are not used in the grammar because they are filtered
  37  * in some intermediate phases. But they still must be declared because
  38  * ocamllex may generate them, or some intermediate phase may also
  39  * generate them (like some functions in parsing_hacks.ml).
  40  * Here we don't have this problem again so we can have a clearer token type.
  41  *
  42  *
  43  *)
  44
  45 (*****************************************************************************)
  46 (* Cpp constructs put it comments in lexer or parsing_hack *)
  47 (*****************************************************************************)
  48
  49 (* history: was in ast_c.ml before:
  50  *  This type is not in the Ast but is associated with the TCommentCpp
  51  *  token. I put this enum here because parser_c.mly need it. I could have put
  52  *  it also in lexer_parser.
  53  *
  54  * update: now in token_c.ml, and actually right now we want those tokens
  55  * to be in the ast so that in the matching/transforming of C code, we
  56  * can detect if some metavariables match code which have some
  57  * cpp_passed tokens next to them (and so where we should issue a warning).
  58  *)
  59 type cppcommentkind =
  60   | CppDirective
  61   | CppAttr
  62   | CppMacro
  63   | CppPassingNormal (* ifdef 0, cplusplus, etc *)
  64   | CppPassingCosWouldGetError (* expr passsing *)
  65   | CppPassingExplicit (* skip_start/end tag *)
  66
  67 (*****************************************************************************)
  68 (* Types *)
  69 (*****************************************************************************)
  70
  71 (*
  72  * TODO? Do we want to handle also non OriginTok-like tokens here ?
  73  * Right now we use this file to be able to later store in the
  74  * ast some information about comments and passed cpp tokens, to
  75  * improve our matching/transforming and unparsing in coccinelle.
  76  * So we should be concerned really only with origin tok, so right
  77  * now I use a simple Common.parse_info, not the more complex
  78  * Ast_c.parse_info, or even more complex Ast_c.info.
  79  * Also right now I defined only the token_tags of comment-like
  80  * tokens.
  81  *)
  82
  83 type info = Common.parse_info
  84
  85 (* I try to be consistent with the names in parser_c.mli *)
  86 type token = token_tag * info
  87  and token_tag =
  88   | TCommentSpace
  89   | TCommentNewline
  90
  91   | TComment
  92
  93   (* the passed tokens because of our limited handling of cpp *)
  94   | TCommentCpp of cppcommentkind
  95
  96   (*| TUnknown ? *)
  97
  98
  99
 100 (* Later if decide to include more kinds of tokens, then may
 101  * have to move the current token_tag like TCommentXxx in their
 102  * own type and have a generic TCommentLike of comment_like_token
 103  * in token_tag. Could also do like in token_helpers have some
 104  * is_xxx predicate, but it's not very pretty (but required when
 105  * some tokens can belong to multiple categories).
 106  *
 107  * It's supposed to be all the tokens that are not otherwise represented
 108  * in the ast via regular constructors and info.
 109  *)
 110 type comment_like_token = token
 111
 112
 113
 114 (*****************************************************************************)
 115 (* Getters *)
 116 (*****************************************************************************)
 117
 118 (* simpler than in token_helpers :) because we don't have the ocamlyacc
 119  * constraints on how to define the token type. *)
 120 let info_of_token = snd
 121
 122
 123
 124 (*****************************************************************************)
 125 (*****************************************************************************)
 126 (* remaining tokens
 127
 128 could define a type  token_class = Comment | Ident | Operator | ...
 129
 130   | TInt of (string * Ast_c.info)
 131   | TFloat of ((string * Ast_c.floatType) * Ast_c.info)
 132   | TChar of ((string * Ast_c.isWchar) * Ast_c.info)
 133   | TString of ((string * Ast_c.isWchar) * Ast_c.info)
 134
 135   | TIdent of (string * Ast_c.info)
 136   | TypedefIdent of (string * Ast_c.info)
 137
 138   | TOPar of (Ast_c.info)
 139   | TCPar of (Ast_c.info)
 140   | TOBrace of (Ast_c.info)
 141   | TCBrace of (Ast_c.info)
 142   | TOCro of (Ast_c.info)
 143   | TCCro of (Ast_c.info)
 144   | TDot of (Ast_c.info)
 145   | TComma of (Ast_c.info)
 146   | TPtrOp of (Ast_c.info)
 147   | TInc of (Ast_c.info)
 148   | TDec of (Ast_c.info)
 149   | TAssign of (Ast_c.assignOp * Ast_c.info)
 150   | TEq of (Ast_c.info)
 151   | TWhy of (Ast_c.info)
 152   | TTilde of (Ast_c.info)
 153   | TBang of (Ast_c.info)
 154   | TEllipsis of (Ast_c.info)
 155   | TDotDot of (Ast_c.info)
 156   | TPtVirg of (Ast_c.info)
 157   | TOrLog of (Ast_c.info)
 158   | TAndLog of (Ast_c.info)
 159   | TOr of (Ast_c.info)
 160   | TXor of (Ast_c.info)
 161   | TAnd of (Ast_c.info)
 162   | TEqEq of (Ast_c.info)
 163   | TNotEq of (Ast_c.info)
 164   | TInf of (Ast_c.info)
 165   | TSup of (Ast_c.info)
 166   | TInfEq of (Ast_c.info)
 167   | TSupEq of (Ast_c.info)
 168   | TShl of (Ast_c.info)
 169   | TShr of (Ast_c.info)
 170   | TPlus of (Ast_c.info)
 171   | TMinus of (Ast_c.info)
 172   | TMul of (Ast_c.info)
 173   | TDiv of (Ast_c.info)
 174   | TMod of (Ast_c.info)
 175   | Tchar of (Ast_c.info)
 176   | Tshort of (Ast_c.info)
 177   | Tint of (Ast_c.info)
 178   | Tdouble of (Ast_c.info)
 179   | Tfloat of (Ast_c.info)
 180   | Tlong of (Ast_c.info)
 181   | Tunsigned of (Ast_c.info)
 182   | Tsigned of (Ast_c.info)
 183   | Tvoid of (Ast_c.info)
 184   | Tauto of (Ast_c.info)
 185   | Tregister of (Ast_c.info)
 186   | Textern of (Ast_c.info)
 187   | Tstatic of (Ast_c.info)
 188   | Ttypedef of (Ast_c.info)
 189   | Tconst of (Ast_c.info)
 190   | Tvolatile of (Ast_c.info)
 191   | Tstruct of (Ast_c.info)
 192   | Tunion of (Ast_c.info)
 193   | Tenum of (Ast_c.info)
 194   | Tbreak of (Ast_c.info)
 195   | Telse of (Ast_c.info)
 196   | Tswitch of (Ast_c.info)
 197   | Tcase of (Ast_c.info)
 198   | Tcontinue of (Ast_c.info)
 199   | Tfor of (Ast_c.info)
 200   | Tdo of (Ast_c.info)
 201   | Tif of (Ast_c.info)
 202   | Twhile of (Ast_c.info)
 203   | Treturn of (Ast_c.info)
 204   | Tgoto of (Ast_c.info)
 205   | Tdefault of (Ast_c.info)
 206   | Tsizeof of (Ast_c.info)
 207   | Trestrict of (Ast_c.info)
 208   | Tasm of (Ast_c.info)
 209   | Tattribute of (Ast_c.info)
 210   | Tinline of (Ast_c.info)
 211   | Ttypeof of (Ast_c.info)
 212
 213   | TDefine of (Ast_c.info)
 214   | TDefParamVariadic of ((string * Ast_c.info))
 215
 216   | TCppEscapedNewline of (Ast_c.info)
 217
 218   | TOParDefine of (Ast_c.info)
 219   | TOBraceDefineInit of (Ast_c.info)
 220   | TIdentDefine of ((string * Ast_c.info))
 221   | TDefEOL of (Ast_c.info)
 222   | TInclude of ((string * string * bool ref * Ast_c.info))
 223   | TIncludeStart of ((Ast_c.info * bool ref))
 224   | TIncludeFilename of ((string * Ast_c.info))
 225   | TIfdef of (((int * int) option ref * Ast_c.info))
 226   | TIfdefelse of (((int * int) option ref * Ast_c.info))
 227   | TIfdefelif of (((int * int) option ref * Ast_c.info))
 228   | TEndif of (((int * int) option ref * Ast_c.info))
 229   | TIfdefBool of ((bool * (int * int) option ref * Ast_c.info))
 230   | TIfdefMisc of ((bool * (int * int) option ref * Ast_c.info))
 231   | TIfdefVersion of ((bool * (int * int) option ref * Ast_c.info))
 232   | TUndef of (string * Ast_c.info)
 233   | TCppDirectiveOther of (Ast_c.info)
 234
 235   | TMacroAttr of ((string * Ast_c.info))
 236   | TMacroStmt of ((string * Ast_c.info))
 237   | TMacroString of ((string * Ast_c.info))
 238   | TMacroDecl of ((string * Ast_c.info))
 239   | TMacroDeclConst of (Ast_c.info)
 240   | TMacroStructDecl of ((string * Ast_c.info))
 241   | TMacroIterator of ((string * Ast_c.info))
 242   | TMacroAttrStorage of ((string * Ast_c.info))
 243
 244   | TCommentSkipTagStart of (Ast_c.info)
 245   | TCommentSkipTagEnd of (Ast_c.info)
 246
 247   | TCParEOL of (Ast_c.info)
 248   | TAction of (Ast_c.info)
 249
 250   | TCommentMisc xxx
 251
 252   | EOF of (Ast_c.info)
 253 *)
 254
 255
 256 (*****************************************************************************)
 257 (* Helpers *)
 258 (*****************************************************************************)