Line data    Source code

       1             : /*
       2             :  * Licensed to the Apache Software Foundation (ASF) under one
       3             :  * or more contributor license agreements. See the NOTICE file
       4             :  * distributed with this work for additional information
       5             :  * regarding copyright ownership. The ASF licenses this file
       6             :  * to you under the Apache License, Version 2.0 (the
       7             :  * "License"); you may not use this file except in compliance
       8             :  * with the License. You may obtain a copy of the License at
       9             :  *
      10             :  *   http://www.apache.org/licenses/LICENSE-2.0
      11             :  *
      12             :  * Unless required by applicable law or agreed to in writing,
      13             :  * software distributed under the License is distributed on an
      14             :  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
      15             :  * KIND, either express or implied. See the License for the
      16             :  * specific language governing permissions and limitations
      17             :  * under the License.
      18             :  */
      19             : 
      20             : /**
      21             :  * Thrift scanner.
      22             :  * #ifdef SANDESH
      23             :  * Sandesh (Message) scanner based on Thrift
      24             :  * #endif 
      25             :  *
      26             :  * Tokenizes a thrift definition file.
      27             :  * #ifdef SANDESH
      28             :  * Tokenizes a sandesh (message) definition file
      29             :  * #endif
      30             :  */
      31             : 
      32             : %{
      33             : 
      34             : #include <cassert>
      35             : #include <string>
      36             : #include <errno.h>
      37             : 
      38             : #include "main.h"
      39             : #include "globals.h"
      40             : #include "parse/t_program.h"
      41             : 
      42             : /**
      43             :  * Must be included AFTER parse/t_program.h, but I can't remember why anymore
      44             :  * because I wrote this a while ago.
      45             :  */
      46             : #ifdef SANDESH 
      47             : #include "sandeshy.hh"
      48             : #else
      49             : #include "thrifty.h"
      50             : #endif
      51             : 
      52           0 : void thrift_reserved_keyword(char* keyword) {
      53           0 :   yyerror("Cannot use reserved language keyword: \"%s\"\n", keyword);
      54           0 :   exit(1);
      55             : }
      56             : 
      57           0 : void integer_overflow(char* text) {
      58           0 :   yyerror("This integer is too big: \"%s\"\n", text);
      59           0 :   exit(1);
      60             : }
      61             : 
      62           0 : void unexpected_token(char* text) {
      63           0 :   yyerror("Unexpected token in input: \"%s\"\n", text);
      64           0 :   exit(1);
      65             : }
      66             : 
      67             : %}
      68             : 
      69             : /**
      70             :  * Provides the yylineno global, useful for debugging output
      71             :  */
      72             : %option lex-compat
      73             : 
      74             : /**
      75             :  * Our inputs are all single files, so no need for yywrap
      76             :  */
      77             : %option noyywrap
      78             : 
      79             : /**
      80             :  * We don't use it, and it fires up warnings at -Wall
      81             :  */
      82             : %option nounput
      83             : 
      84             : /**
      85             :  * Helper definitions, comments, constants, and whatnot
      86             :  */
      87             : 
      88             : intconstant   ([+-]?[0-9]+)
      89             : hexconstant   ("0x"[0-9A-Fa-f]+)
      90             : dubconstant   ([+-]?[0-9]*(\.[0-9]+)?([eE][+-]?[0-9]+)?)
      91             : identifier    ([a-zA-Z_][\.a-zA-Z_0-9]*)
      92             : whitespace    ([ \t\r\n]*)
      93             : sillycomm     ("/*""*"*"*/")
      94             : multicomm     ("/*"[^*]"/"*([^*/]|[^*]"/"|"*"[^/])*"*"*"*/")
      95             : doctext       ("/**"([^*/]|[^*]"/"|"*"[^/])*"*"*"*/")
      96             : comment       ("//"[^\n]*)
      97             : unixcomment   ("#"[^\n]*)
      98             : symbol        ([:;\,\{\}\(\)\=<>\[\]])
      99             : st_identifier ([a-zA-Z-][\.a-zA-Z_0-9-]*)
     100             : literal_begin (['\"])
     101             : format_str    (%([1-9]$)?([+-]?[ -#0])?([1-9]+)?(\.[1-9]+)?(hh|h|l|ll|L|z|j|t)?(d|u|f|F|e|E|g|G|x|X|o|s|c))
     102             : 
     103             : %%
     104             : 
     105             : {whitespace}         { /* do nothing */                 }
     106      112448 : {sillycomm}          { /* do nothing */                 }
     107           0 : {multicomm}          { /* do nothing */                 }
     108         972 : {comment}            { /* do nothing */                 }
     109        3072 : {unixcomment}        { /* do nothing */                 }
     110           0 : 
     111       62036 : {symbol}             { return yytext[0];                }
     112           0 : "*"                  { return yytext[0];                }
     113           0 : 
     114           0 : "false"              { yylval.iconst=0; return tok_int_constant; }
     115           0 : "true"               { yylval.iconst=1; return tok_int_constant; }
     116           0 : 
     117          60 : "namespace"          { return tok_namespace;            }
     118           0 : "cpp_namespace"      { return tok_cpp_namespace;        }
     119           0 : "cpp_include"        { return tok_cpp_include;          }
     120           0 : "cpp_type"           { return tok_cpp_type;             }
     121           0 : "java_package"       { return tok_java_package;         }
     122           0 : "cocoa_prefix"       { return tok_cocoa_prefix;         }
     123           0 : "csharp_namespace"   { return tok_csharp_namespace;     }
     124           0 : "delphi_namespace"   { return tok_delphi_namespace;     }
     125           0 : "php_namespace"      { return tok_php_namespace;        }
     126           0 : "py_module"          { return tok_py_module;            }
     127           0 : "perl_package"       { return tok_perl_package;         }
     128           0 : "ruby_namespace"     { return tok_ruby_namespace;       }
     129           0 : "smalltalk_category" { return tok_smalltalk_category;   }
     130           0 : "smalltalk_prefix"   { return tok_smalltalk_prefix;     }
     131           0 : "xsd_all"            { return tok_xsd_all;              }
     132           0 : "xsd_optional"       { return tok_xsd_optional;         }
     133           0 : "xsd_nillable"       { return tok_xsd_nillable;         }
     134           0 : "xsd_namespace"      { return tok_xsd_namespace;        }
     135           0 : "xsd_attrs"          { return tok_xsd_attrs;            }
     136           0 : "include"            { return tok_include;              }
     137          72 : "void"               { return tok_void;                 }
     138           0 : "bool"               { return tok_bool;                 }
     139         880 : "byte"               { return tok_byte;                 }
     140         108 : "i16"                { return tok_i16;                  }
     141          48 : "i32"                { return tok_i32;                  }
     142         380 : "i64"                { return tok_i64;                  }
     143         276 : "u16"                { return tok_u16;                  }
     144         948 : "u32"                { return tok_u32;                  }
     145        1152 : "u64"                { return tok_u64;                  }
     146        1632 : "ipv4"               { return tok_ipv4;                 }
     147          24 : "ipaddr"             { return tok_ipaddr;               }
     148          96 : "double"             { return tok_double;               }
     149         716 : "string"             { return tok_string;               }
     150        9936 : "binary"             { return tok_binary;               }
     151           0 : "xml"                { return tok_xml;                  }
     152         108 : "ct_uuid_t"          { return tok_uuid_t;               }
     153         108 : "slist"              { return tok_slist;                }
     154           0 : "senum"              { return tok_senum;                }
     155           0 : "map"                { return tok_map;                  }
     156         976 : "list"               { return tok_list;                 }
     157        1648 : "set"                { return tok_set;                  }
     158          48 : "oneway"             { return tok_oneway;               }
     159           0 : "typedef"            { return tok_typedef;              }
     160           0 : "struct"             { return tok_struct;               }
     161        2128 : "union"              { return tok_union;                }
     162          12 : "exception"          { return tok_xception;             }
     163           0 : "extends"            { return tok_extends;              }
     164           0 : "throws"             { return tok_throws;               }
     165           0 : "service"            { return tok_service;              }
     166           0 : "enum"               { return tok_enum;                 }
     167         396 : "const"              { return tok_const;                }
     168        2544 : "required"           { return tok_required;             }
     169           0 : "optional"           { return tok_optional;             }
     170        4028 : "systemlog"          { return tok_system;               }
     171         376 : "request"            { return tok_request;              }
     172         552 : "response"           { return tok_response;             }
     173         456 : "trace"              { return tok_trace;                }
     174         324 : "traceobject"        { return tok_trace_object;         }
     175          12 : "objectlog"          { return tok_object;               }
     176         124 : "flowlog"            { return tok_flow;                 } 
     177           0 : "sessionlog"         { return tok_session;              }
     178          12 : "buffer"             { return tok_buffer;               }
     179           0 : "uve"                { return tok_uve;                  }
     180         256 : "dynamicuve"         { return tok_dynamic_uve;          }
     181          36 : "alarm"              { return tok_alarm;                }
     182          12 : "sandesh"            { return tok_sandesh;              }
     183        2160 : "staticfunction"     { return tok_staticfunction;       }
     184           0 : "abcfunction"        { return tok_abcfunction;          }
     185           0 : 
     186           0 : 
     187           0 : "BEGIN"              { thrift_reserved_keyword(yytext); }
     188           0 : "END"                { thrift_reserved_keyword(yytext); }
     189           0 : "__CLASS__"          { thrift_reserved_keyword(yytext); }
     190           0 : "__DIR__"            { thrift_reserved_keyword(yytext); }
     191           0 : "__FILE__"           { thrift_reserved_keyword(yytext); }
     192           0 : "__FUNCTION__"       { thrift_reserved_keyword(yytext); }
     193           0 : "__LINE__"           { thrift_reserved_keyword(yytext); }
     194           0 : "__METHOD__"         { thrift_reserved_keyword(yytext); }
     195           0 : "__NAMESPACE__"      { thrift_reserved_keyword(yytext); }
     196           0 : "abstract"           { thrift_reserved_keyword(yytext); }
     197           0 : "alias"              { thrift_reserved_keyword(yytext); }
     198           0 : "and"                { thrift_reserved_keyword(yytext); }
     199           0 : "args"               { thrift_reserved_keyword(yytext); }
     200           0 : "as"                 { thrift_reserved_keyword(yytext); }
     201           0 : "assert"             { thrift_reserved_keyword(yytext); }
     202           0 : "begin"              { thrift_reserved_keyword(yytext); }
     203           0 : "break"              { thrift_reserved_keyword(yytext); }
     204           0 : "case"               { thrift_reserved_keyword(yytext); }
     205           0 : "catch"              { thrift_reserved_keyword(yytext); }
     206           0 : "class"              { thrift_reserved_keyword(yytext); }
     207           0 : "clone"              { thrift_reserved_keyword(yytext); }
     208           0 : "continue"           { thrift_reserved_keyword(yytext); }
     209           0 : "declare"            { thrift_reserved_keyword(yytext); }
     210           0 : "def"                { thrift_reserved_keyword(yytext); }
     211           0 : "default"            { thrift_reserved_keyword(yytext); }
     212           0 : "del"                { thrift_reserved_keyword(yytext); }
     213           0 : "delete"             { thrift_reserved_keyword(yytext); }
     214           0 : "do"                 { thrift_reserved_keyword(yytext); }
     215           0 : "dynamic"            { thrift_reserved_keyword(yytext); }
     216           0 : "elif"               { thrift_reserved_keyword(yytext); }
     217           0 : "else"               { thrift_reserved_keyword(yytext); }
     218           0 : "elseif"             { thrift_reserved_keyword(yytext); }
     219           0 : "elsif"              { thrift_reserved_keyword(yytext); }
     220           0 : "end"                { thrift_reserved_keyword(yytext); }
     221           0 : "enddeclare"         { thrift_reserved_keyword(yytext); }
     222           0 : "endfor"             { thrift_reserved_keyword(yytext); }
     223           0 : "endforeach"         { thrift_reserved_keyword(yytext); }
     224           0 : "endif"              { thrift_reserved_keyword(yytext); }
     225           0 : "endswitch"          { thrift_reserved_keyword(yytext); }
     226           0 : "endwhile"           { thrift_reserved_keyword(yytext); }
     227           0 : "ensure"             { thrift_reserved_keyword(yytext); }
     228           0 : "except"             { thrift_reserved_keyword(yytext); }
     229           0 : "exec"               { thrift_reserved_keyword(yytext); }
     230           0 : "finally"            { thrift_reserved_keyword(yytext); }
     231           0 : "float"              { thrift_reserved_keyword(yytext); }
     232           0 : "for"                { thrift_reserved_keyword(yytext); }
     233           0 : "foreach"            { thrift_reserved_keyword(yytext); }
     234           0 : "function"           { thrift_reserved_keyword(yytext); }
     235           0 : "global"             { thrift_reserved_keyword(yytext); }
     236           0 : "goto"               { thrift_reserved_keyword(yytext); }
     237           0 : "if"                 { thrift_reserved_keyword(yytext); }
     238           0 : "implements"         { thrift_reserved_keyword(yytext); }
     239           0 : "import"             { thrift_reserved_keyword(yytext); }
     240           0 : "in"                 { thrift_reserved_keyword(yytext); }
     241           0 : "inline"             { thrift_reserved_keyword(yytext); }
     242           0 : "instanceof"         { thrift_reserved_keyword(yytext); }
     243           0 : "interface"          { thrift_reserved_keyword(yytext); }
     244           0 : "is"                 { thrift_reserved_keyword(yytext); }
     245           0 : "lambda"             { thrift_reserved_keyword(yytext); }
     246           0 : "module"             { thrift_reserved_keyword(yytext); }
     247           0 : "native"             { thrift_reserved_keyword(yytext); }
     248           0 : "new"                { thrift_reserved_keyword(yytext); }
     249           0 : "next"               { thrift_reserved_keyword(yytext); }
     250           0 : "nil"                { thrift_reserved_keyword(yytext); }
     251           0 : "not"                { thrift_reserved_keyword(yytext); }
     252           0 : "or"                 { thrift_reserved_keyword(yytext); }
     253           0 : "pass"               { thrift_reserved_keyword(yytext); }
     254           0 : "public"             { thrift_reserved_keyword(yytext); }
     255           0 : "print"              { thrift_reserved_keyword(yytext); }
     256           0 : "private"            { thrift_reserved_keyword(yytext); }
     257           0 : "protected"          { thrift_reserved_keyword(yytext); }
     258           0 : "public"             { thrift_reserved_keyword(yytext); }
     259           0 : "raise"              { thrift_reserved_keyword(yytext); }
     260           0 : "redo"               { thrift_reserved_keyword(yytext); }
     261           0 : "rescue"             { thrift_reserved_keyword(yytext); }
     262           0 : "retry"              { thrift_reserved_keyword(yytext); }
     263           0 : "register"           { thrift_reserved_keyword(yytext); }
     264           0 : "return"             { thrift_reserved_keyword(yytext); }
     265           0 : "self"               { thrift_reserved_keyword(yytext); }
     266           0 : "sizeof"             { thrift_reserved_keyword(yytext); }
     267           0 : "static"             { thrift_reserved_keyword(yytext); }
     268           0 : "super"              { thrift_reserved_keyword(yytext); }
     269           0 : "switch"             { thrift_reserved_keyword(yytext); }
     270           0 : "synchronized"       { thrift_reserved_keyword(yytext); }
     271           0 : "then"               { thrift_reserved_keyword(yytext); }
     272           0 : "this"               { thrift_reserved_keyword(yytext); }
     273           0 : "throw"              { thrift_reserved_keyword(yytext); }
     274           0 : "transient"          { thrift_reserved_keyword(yytext); }
     275           0 : "try"                { thrift_reserved_keyword(yytext); }
     276           0 : "undef"              { thrift_reserved_keyword(yytext); }
     277           0 : "union"              { thrift_reserved_keyword(yytext); }
     278           0 : "unless"             { thrift_reserved_keyword(yytext); }
     279           0 : "unsigned"           { thrift_reserved_keyword(yytext); }
     280           0 : "until"              { thrift_reserved_keyword(yytext); }
     281           0 : "use"                { thrift_reserved_keyword(yytext); }
     282           0 : "var"                { thrift_reserved_keyword(yytext); }
     283           0 : "virtual"            { thrift_reserved_keyword(yytext); }
     284           0 : "volatile"           { thrift_reserved_keyword(yytext); }
     285           0 : "when"               { thrift_reserved_keyword(yytext); }
     286           0 : "while"              { thrift_reserved_keyword(yytext); }
     287           0 : "with"               { thrift_reserved_keyword(yytext); }
     288           0 : "xor"                { thrift_reserved_keyword(yytext); }
     289           0 : "yield"              { thrift_reserved_keyword(yytext); }
     290           0 : 
     291       16980 : {intconstant} {
     292       16980 :   errno = 0;
     293       16980 :   yylval.iconst = strtoll(yytext, NULL, 10);
     294       16980 :   if (errno == ERANGE) {
     295           0 :     integer_overflow(yytext);
     296             :   }
     297       16980 :   return tok_int_constant;
     298             : }
     299             : 
     300          48 : {hexconstant} {
     301          48 :   errno = 0;
     302          48 :   yylval.iconst = strtoll(yytext+2, NULL, 16);
     303          48 :   if (errno == ERANGE) {
     304           0 :     integer_overflow(yytext);
     305             :   }
     306          48 :   return tok_int_constant;
     307             : }
     308             : 
     309           0 : {dubconstant} {
     310           0 :   yylval.dconst = atof(yytext);
     311           0 :   return tok_dub_constant;
     312             : }
     313             : 
     314       36360 : {identifier} {
     315       36360 :   yylval.id = strdup(yytext);
     316       36360 :   return tok_identifier;
     317             : }
     318             : 
     319           0 : {st_identifier} {
     320           0 :   yylval.id = strdup(yytext);
     321           0 :   return tok_st_identifier;
     322             : }
     323             : 
     324        5316 : {literal_begin} {
     325        5316 :   char mark = yytext[0];
     326        5316 :   std::string result;
     327             :   for(;;)
     328             :   {
     329       76352 :     int ch = yyinput();
     330       76352 :     switch (ch) {
     331           0 :       case EOF:
     332           0 :         yyerror("End of file while read string at %d\n", yylineno);
     333           0 :         exit(1);
     334           0 :       case '\n':
     335           0 :         yyerror("End of line while read string at %d\n", yylineno - 1);
     336           0 :         exit(1);
     337           0 :       case '\\':
     338           0 :         ch = yyinput();
     339           0 :         switch (ch) {
     340           0 :           case 'r':
     341           0 :             result.push_back('\r');
     342           0 :             continue;
     343           0 :           case 'n':
     344           0 :             result.push_back('\n');
     345           0 :             continue;
     346           0 :           case 't':
     347           0 :             result.push_back('\t');
     348           0 :             continue;
     349           0 :           case '"':
     350           0 :             result.push_back('"');
     351           0 :             continue;
     352           0 :           case '\'':
     353           0 :             result.push_back('\'');
     354           0 :             continue;
     355           0 :           case '\\':
     356           0 :             result.push_back('\\');
     357           0 :             continue;
     358           0 :           default:
     359           0 :             yyerror("Bad escape character\n");
     360           0 :             return -1;
     361             :         }
     362             :         break;
     363       76352 :       default:
     364       76352 :         if (ch == mark) {
     365        5316 :           yylval.id = strdup(result.c_str());
     366        5316 :           return tok_literal;
     367             :         } else {
     368       71036 :           result.push_back(ch);
     369             :         }
     370             :     }
     371       71036 :   }
     372        5316 : }
     373             : 
     374        3268 : 
     375             : {doctext} {
     376             :  /* This does not show up in the parse tree. */
     377             :  /* Rather, the parser will grab it out of the global. */
     378        3268 :   if (g_parse_mode == PROGRAM) {
     379        1634 :     clear_doctext();
     380        1634 :     g_doctext = strdup(yytext + 3);
     381        1634 :     assert(strlen(g_doctext) >= 2);
     382        1634 :     g_doctext[strlen(g_doctext) - 2] = ' ';
     383        1634 :     g_doctext[strlen(g_doctext) - 1] = '\0';
     384        1634 :     g_doctext = clean_up_doctext(g_doctext);
     385        1634 :     g_doctext_lineno = yylineno;
     386        1634 :     if( (g_program_doctext_candidate == NULL) && (g_program_doctext_status == INVALID)){
     387         164 :       g_program_doctext_candidate = strdup(g_doctext);
     388         164 :       g_program_doctext_lineno = g_doctext_lineno;
     389         164 :       g_program_doctext_status = STILL_CANDIDATE;
     390         164 :       pdebug("%s","program doctext set to STILL_CANDIDATE");
     391             :     }
     392             :   }
     393             : }
     394        3268 : 
     395           0 : {format_str} {
     396           0 :   yylval.id = strdup(yytext);
     397           0 :   return tok_format;
     398             : }
     399             : 
     400           0 : . {
     401           0 :   unexpected_token(yytext);
     402             : }
     403           0 : 
     404           0 : 
     405             : . {
     406             :   /* Catch-all to let us catch "*" in the parser. */
     407           0 :   return (int) yytext[0];
     408             : }
     409             : 
     410           0 : %%
     411           0 : 
     412             : /* vim: filetype=lex
     413             : */