diff options
Diffstat (limited to 'debian/uncrustify-trinity/uncrustify-trinity-0.74.0/scripts/tokenizer.py')
| -rwxr-xr-x | debian/uncrustify-trinity/uncrustify-trinity-0.74.0/scripts/tokenizer.py | 316 |
1 files changed, 316 insertions, 0 deletions
diff --git a/debian/uncrustify-trinity/uncrustify-trinity-0.74.0/scripts/tokenizer.py b/debian/uncrustify-trinity/uncrustify-trinity-0.74.0/scripts/tokenizer.py new file mode 100755 index 00000000..0bc33bac --- /dev/null +++ b/debian/uncrustify-trinity/uncrustify-trinity-0.74.0/scripts/tokenizer.py @@ -0,0 +1,316 @@ +#! /usr/bin/env python +# tokenize.py +# +# Parses a C/C++/C#/D/Java/Pawn/whatever file in an array of +# tuples (string, type) +# + +# punctuator lookup table +punc_table = [ + [ '!', 25, 26, '!' ], # 0: '!' + [ '#', 24, 35, '#' ], # 1: '#' + [ '$', 23, 0, '$' ], # 2: '$' + [ '%', 22, 36, '%' ], # 3: '%' + [ '&', 21, 41, '&' ], # 4: '&' + [ '(', 20, 0, '(' ], # 5: '(' + [ ')', 19, 0, ')' ], # 6: ')' + [ '*', 18, 43, '*' ], # 7: '*' + [ '+', 17, 44, '+' ], # 8: '+' + [ ',', 16, 0, ',' ], # 9: ',' + [ '-', 15, 46, '-' ], # 10: '-' + [ '.', 14, 50, '.' ], # 11: '.' + [ '/', 13, 53, '/' ], # 12: '/' + [ ':', 12, 54, ':' ], # 13: ':' + [ ';', 11, 0, ';' ], # 14: ';' + [ '<', 10, 56, '<' ], # 15: '<' + [ '=', 9, 63, '=' ], # 16: '=' + [ '>', 8, 65, '>' ], # 17: '>' + [ '?', 7, 0, '?' ], # 18: '?' + [ '[', 6, 70, '[' ], # 19: '[' + [ ']', 5, 0, ']' ], # 20: ']' + [ '^', 4, 71, '^' ], # 21: '^' + [ '{', 3, 0, '{' ], # 22: '{' + [ '|', 2, 72, '|' ], # 23: '|' + [ '}', 1, 0, '}' ], # 24: '}' + [ '~', 0, 74, '~' ], # 25: '~' + [ '<', 3, 30, '!<' ], # 26: '!<' + [ '=', 2, 33, '!=' ], # 27: '!=' + [ '>', 1, 34, '!>' ], # 28: '!>' + [ '~', 0, 0, '!~' ], # 29: '!~' + [ '=', 1, 0, '!<=' ], # 30: '!<=' + [ '>', 0, 32, '!<>' ], # 31: '!<>' + [ '=', 0, 0, '!<>='], # 32: '!<>=' + [ '=', 0, 0, '!==' ], # 33: '!==' + [ '=', 0, 0, '!>=' ], # 34: '!>=' + [ '#', 0, 0, '##' ], # 35: '##' + [ ':', 2, 39, '%:' ], # 36: '%:' + [ '=', 1, 0, '%=' ], # 37: '%=' + [ '>', 0, 0, '%>' ], # 38: '%>' + [ '%', 0, 40, None ], # 39: '%:%' + [ ':', 0, 0, '%:%:'], # 40: '%:%:' + [ '&', 1, 0, '&&' ], # 41: '&&' + [ '=', 0, 0, '&=' ], # 42: '&=' + [ '=', 0, 0, '*=' ], # 43: '*=' + [ '+', 1, 0, '++' ], # 44: '++' + [ '=', 0, 0, '+=' ], # 45: '+=' + [ '-', 2, 0, '--' ], # 46: '--' + [ '=', 1, 0, '-=' ], # 47: '-=' + [ '>', 0, 49, '->' ], # 48: '->' + [ '*', 0, 0, '->*' ], # 49: '->*' + [ '*', 1, 0, '.*' ], # 50: '.*' + [ '.', 0, 52, '..' ], # 51: '..' + [ '.', 0, 0, '...' ], # 52: '...' + [ '=', 0, 0, '/=' ], # 53: '/=' + [ ':', 1, 0, '::' ], # 54: '::' + [ '>', 0, 0, ':>' ], # 55: ':>' + [ '%', 4, 0, '<%' ], # 56: '<%' + [ ':', 3, 0, '<:' ], # 57: '<:' + [ '<', 2, 61, '<<' ], # 58: '<<' + [ '=', 1, 0, '<=' ], # 59: '<=' + [ '>', 0, 62, '<>' ], # 60: '<>' + [ '=', 0, 0, '<<=' ], # 61: '<<=' + [ '=', 0, 0, '<>=' ], # 62: '<>=' + [ '=', 0, 64, '==' ], # 63: '==' + [ '=', 0, 0, '===' ], # 64: '===' + [ '=', 1, 0, '>=' ], # 65: '>=' + [ '>', 0, 67, '>>' ], # 66: '>>' + [ '=', 1, 0, '>>=' ], # 67: '>>=' + [ '>', 0, 69, '>>>' ], # 68: '>>>' + [ '=', 0, 0, '>>>='], # 69: '>>>=' + [ ']', 0, 0, '[]' ], # 70: '[]' + [ '=', 0, 0, '^=' ], # 71: '^=' + [ '=', 1, 0, '|=' ], # 72: '|=' + [ '|', 0, 0, '||' ], # 73: '||' + [ '=', 1, 0, '~=' ], # 74: '~=' + [ '~', 0, 0, '~~' ], # 75: '~~' +] + + +# +# Token types: +# 0 = newline +# 1 = punctuator +# 2 = integer +# 3 = float +# 4 = string +# 5 = identifier +# +class Tokenizer: + def __init__(self): + self.tokens = [] + self.text = '' + self.text_idx = 0 + + def tokenize_text(self, in_text): + self.tokens = [] + self.text = in_text + self.text_idx = 0 + + print(in_text) + try: + while self.text_idx < len(self.text): + if self.parse_whitespace(): + continue + elif self.text[self.text_idx] == '\\' and self.text[self.text_idx + 1] == '\n': + self.text_idx += 2 + continue + elif self.parse_comment(): + continue + elif self.parse_number(): + continue + elif self.parse_identifier(): + continue + elif self.parse_string(): + continue + elif self.parse_punctuator(): + continue + else: + print("confused: %s" % self.text[self.text_idx:]) + break + except: + print("bombed") + raise + + def parse_whitespace(self): + start_idx = self.text_idx + hit_newline = False + while self.text_idx < len(self.text): + if self.text[self.text_idx] in '\n\r': + hit_newline = True + elif not self.text[self.text_idx] in ' \t': + break + self.text_idx += 1 + + if hit_newline: + self.tokens.append(('\n', 0)) + return start_idx != self.text_idx + + def parse_comment(self): + if not self.text[self.text_idx] == '/' or not self.text[self.text_idx + 1] in '/*': + return False + if self.text[self.text_idx + 1] == '/': + while self.text_idx < len(self.text): + if self.text[self.text_idx] in '\n\r': + break + self.text_idx += 1 + else: + while self.text_idx < len(self.text) - 1: + if self.text[self.text_idx] == '*' and self.text[self.text_idx + 1] == '/': + self.text_idx += 2 + break + self.text_idx += 1 + return True + + def parse_identifier(self): + if not self.text[self.text_idx].upper() in '@_ABCDEFGHIJKLMNOPQRSTUVWXYZ': + return False + start_idx = self.text_idx + while self.text_idx < len(self.text) and \ + self.text[self.text_idx].upper() in '@_ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890': + self.text_idx += 1 + self.tokens.append((self.text[start_idx : self.text_idx], 5)) + return True + + def parse_string(self): + starter = 0 + start_ch = self.text[self.text_idx] + if start_ch == 'L': + starter = 1 + start_ch = self.text[self.text_idx + 1] + if not start_ch in '"\'': + return False + start_idx = self.text_idx + self.text_idx += starter + 1 + escaped = False + while self.text_idx < len(self.text): + if escaped: + escaped = False + else: + if self.text[self.text_idx] == '\\': + escaped = True + elif self.text[self.text_idx] == start_ch: + self.text_idx += 1 + break + self.text_idx += 1 + + self.tokens.append((self.text[start_idx : self.text_idx], 4)) + return True + + # Checks for punctuators + # Returns whether a punctuator was consumed (True or False) + def parse_punctuator(self): + tab_idx = 0 + punc_len = 0 + saved_punc = None + while 1: + pte = punc_table[tab_idx] + if pte[0] == self.text[self.text_idx]: + if pte[3] is not None: + saved_punc = pte[3] + self.text_idx += 1 + tab_idx = pte[2] + if tab_idx == 0: + break + elif pte[1] == 0: + break + else: + tab_idx += 1 + if saved_punc is not None: + self.tokens.append((saved_punc, 1)) + return True + return False + + def parse_number(self): + # A number must start with a digit or a dot followed by a digit + ch = self.text[self.text_idx] + if not ch.isdigit() and (ch != '.' or not self.text[self.text_idx + 1].isdigit()): + return False + token_type = 2 # integer + if ch == '.': + token_type = 3 # float + did_hex = False + start_idx = self.text_idx + + # Check for Hex, Octal, or Binary + # Note that only D and Pawn support binary, but who cares? + # + if ch == '0': + self.text_idx += 1 + ch = self.text[self.text_idx].upper() + if ch == 'X': # hex + did_hex = True + self.text_idx += 1 + while self.text[self.text_idx] in '_0123456789abcdefABCDEF': + self.text_idx += 1 + elif ch == 'B': # binary + self.text_idx += 1 + while self.text[self.text_idx] in '_01': + self.text_idx += 1 + elif ch >= '0' and ch <= 7: # octal (but allow decimal) + self.text_idx += 1 + while self.text[self.text_idx] in '_0123456789': + self.text_idx += 1 + else: + # either just 0 or 0.1 or 0UL, etc + pass + else: + # Regular int or float + while self.text[self.text_idx] in '_0123456789': + self.text_idx += 1 + + # Check if we stopped on a decimal point + if self.text[self.text_idx] == '.': + self.text_idx += 1 + token_type = 3 # float + if did_hex: + while self.text[self.text_idx] in '_0123456789abcdefABCDEF': + self.text_idx += 1 + else: + while self.text[self.text_idx] in '_0123456789': + self.text_idx += 1 + + # Check exponent + # Valid exponents per language (not that it matters): + # C/C++/D/Java: eEpP + # C#/Pawn: eE + if self.text[self.text_idx] in 'eEpP': + token_type = 3 # float + self.text_idx += 1 + if self.text[self.text_idx] in '+-': + self.text_idx += 1 + while self.text[self.text_idx] in '_0123456789': + self.text_idx += 1 + + # Check the suffixes + # Valid suffixes per language (not that it matters): + # Integer Float + # C/C++: uUlL lLfF + # C#: uUlL fFdDMm + # D: uUL ifFL + # Java: lL fFdD + # Pawn: (none) (none) + # + # Note that i, f, d, and m only appear in floats. + while 1: + if self.text[self.text_idx] in 'tTfFdDmM': + token_type = 3 # float + elif not self.text[self.text_idx] in 'lLuU': + break + self.text_idx += 1 + + self.tokens.append((self.text[start_idx : self.text_idx], token_type)) + return True + +text = """ +1.23+4-3*16%2 *sin(1.e-3 + .5p32) "hello" and "hello\\"there" +123 // some comment +a = b + c; +#define abc \\ + 5 +d = 5 /* hello */ + 3; +""" + +t = Tokenizer() +t.tokenize_text(text) +print(t.tokens) + |