diff options
Diffstat (limited to 'debian/pyrex/pyrex-0.9.9/Pyrex/Compiler/Lexicon.py')
-rw-r--r-- | debian/pyrex/pyrex-0.9.9/Pyrex/Compiler/Lexicon.py | 145 |
1 files changed, 145 insertions, 0 deletions
diff --git a/debian/pyrex/pyrex-0.9.9/Pyrex/Compiler/Lexicon.py b/debian/pyrex/pyrex-0.9.9/Pyrex/Compiler/Lexicon.py new file mode 100644 index 00000000..ca303c0d --- /dev/null +++ b/debian/pyrex/pyrex-0.9.9/Pyrex/Compiler/Lexicon.py @@ -0,0 +1,145 @@ +# +# Pyrex Scanner - Lexical Definitions +# +# Changing anything in this file will cause Lexicon.pickle +# to be rebuilt next time pyrexc is run. +# + +string_prefixes = "cCrR" + +def make_lexicon(): + from Pyrex.Plex import \ + Str, Any, AnyBut, AnyChar, Rep, Rep1, Opt, Bol, Eol, Eof, \ + TEXT, IGNORE, State, Lexicon + from Scanning import Method + + letter = Any("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_") + digit = Any("0123456789") + octdigit = Any("01234567") + hexdigit = Any("0123456789ABCDEFabcdef") + indentation = Bol + Rep(Any(" \t")) + + decimal = Rep1(digit) + dot = Str(".") + exponent = Any("Ee") + Opt(Any("+-")) + decimal + decimal_fract = (decimal + dot + Opt(decimal)) | (dot + decimal) + + name = letter + Rep(letter | digit) + intconst = decimal | (Str("0x") + Rep1(hexdigit)) + longconst = intconst + Str("L") + fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent) + imagconst = (intconst | fltconst) + Any("jJ") + +# sq_string = ( +# Str("'") + +# Rep(AnyBut("\\\n'") | (Str("\\") + AnyChar)) + +# Str("'") +# ) +# +# dq_string = ( +# Str('"') + +# Rep(AnyBut('\\\n"') | (Str("\\") + AnyChar)) + +# Str('"') +# ) +# +# non_sq = AnyBut("'") | (Str('\\') + AnyChar) +# tsq_string = ( +# Str("'''") +# + Rep(non_sq | (Str("'") + non_sq) | (Str("''") + non_sq)) +# + Str("'''") +# ) +# +# non_dq = AnyBut('"') | (Str('\\') + AnyChar) +# tdq_string = ( +# Str('"""') +# + Rep(non_dq | (Str('"') + non_dq) | (Str('""') + non_dq)) +# + Str('"""') +# ) +# +# stringlit = Opt(Any(string_prefixes)) + (sq_string | dq_string | tsq_string| tdq_string) + + beginstring = Opt(Any(string_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""')) + two_oct = octdigit + octdigit + three_oct = octdigit + octdigit + octdigit + two_hex = hexdigit + hexdigit + escapeseq = Str("\\") + (two_oct | three_oct | two_hex | AnyChar) + + bra = Any("([{") + ket = Any(")]}") + punct = Any(":,;+-*/|&<>=.%`~^?") + diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", + "+=", "-=", "*=", "/=", "%=", "**=", "<<=", ">>=", "&=", "^=", "|=") + spaces = Rep1(Any(" \t\f")) + comment = Str("#") + Rep(AnyBut("\n")) + escaped_newline = Str("\\\n") + lineterm = Eol + Opt(Str("\n")) + + return Lexicon([ + (name, 'IDENT'), + (intconst, 'INT'), + (longconst, 'LONG'), + (fltconst, 'FLOAT'), + (imagconst, 'IMAG'), + (punct | diphthong, TEXT), + + (bra, Method('open_bracket_action')), + (ket, Method('close_bracket_action')), + (lineterm, Method('newline_action')), + + #(stringlit, 'STRING'), + (beginstring, Method('begin_string_action')), + + (comment, IGNORE), + (spaces, IGNORE), + (escaped_newline, IGNORE), + + State('INDENT', [ + (Opt(spaces) + Opt(comment) + lineterm, IGNORE), + (indentation, Method('indentation_action')), + (Eof, Method('eof_action')) + ]), + + State('SQ_STRING', [ + (escapeseq, 'ESCAPE'), + (Rep1(AnyBut("'\"\n\\")), 'CHARS'), + (Str('"'), 'CHARS'), + (Str("\n"), Method('unclosed_string_action')), + (Str("'"), Method('end_string_action')), + (Eof, 'EOF') + ]), + + State('DQ_STRING', [ + (escapeseq, 'ESCAPE'), + (Rep1(AnyBut('"\n\\')), 'CHARS'), + (Str("'"), 'CHARS'), + (Str("\n"), Method('unclosed_string_action')), + (Str('"'), Method('end_string_action')), + (Eof, 'EOF') + ]), + + State('TSQ_STRING', [ + (escapeseq, 'ESCAPE'), + (Rep1(AnyBut("'\"\n\\")), 'CHARS'), + (Any("'\""), 'CHARS'), + (Str("\n"), 'NEWLINE'), + (Str("'''"), Method('end_string_action')), + (Eof, 'EOF') + ]), + + State('TDQ_STRING', [ + (escapeseq, 'ESCAPE'), + (Rep1(AnyBut('"\'\n\\')), 'CHARS'), + (Any("'\""), 'CHARS'), + (Str("\n"), 'NEWLINE'), + (Str('"""'), Method('end_string_action')), + (Eof, 'EOF') + ]), + + (Eof, Method('eof_action')) + ], + + # FIXME: Plex 1.9 needs different args here from Plex 1.1.4 + #debug_flags = scanner_debug_flags, + #debug_file = scanner_dump_file + ) + |