/+ Copyright (c) 1999-2006 by Digital Mars All Rights Reserved written by Walter Bright www.digitalmars.com License for redistribution is by either the Artistic License in artistic.txt, or the GNU General Public License in gnu.txt. See the included readme.txt for details. D Language conversion by: J Duncan +/ /** d language lexer */ module dparser.Lexer; import dparser.Root; import dparser.Tokens; import dparser.Token; import dparser.Keyword; import dparser.Types; import dparser.Module; import dparser.Identifier; import dparser.unialpha; import dparser.OutBuffer; //private import std.ctype; //private import std.string; //import dwf.core.debugapi; int errno = 0; //#if _WIN32 && __DMC__ // from \dm\src\include\setlocal.h //extern "C" char * __cdecl __locale_decpoint; char *__locale_decpoint; //#endif //const uint LS = 0x2028; // UTF line separator //const uint PS = 0x2029; // UTF paragraph separator //extern int isUniAlpha(unsigned u); //extern int HtmlNamedEntity(unsigned char *p, int length); /** Lexer object */ class Lexer { static Identifier[char[]] stringtable; static OutBuffer stringbuffer; static Token *freelist; Token token; // current token Module mod; // current module Loc loc; // for error messages ubyte *base; // pointer to start of buffer ubyte *end; // past end of buffer ubyte *p; // current character int doDocComment; // collect doc comment information int anyToken; // !=0 means seen at least one token int commentToken; // !=0 means comments are TOKcomment's this(Module mod, ubyte *base, uint begoffset, uint endoffset, int doDocComment, int commentToken) { if (stringbuffer is null) stringbuffer = new OutBuffer; loc = Loc(mod, 1); this.base = base; this.end = base + endoffset; this.p = base + begoffset; this.mod = mod; this.doDocComment = doDocComment; this.commentToken = commentToken; /* If first line starts with '#!', ignore the line */ if (p[0] == '#' && p[1] == '!') { p += 2; while (true) { ubyte c = *p; switch (c) { case '\n': p++; break; case '\r': p++; if (*p == '\n') p++; break; case 0: case 0x1A: break; default: if (c & 0x80) { uint u = decodeUTF(); if (u == PS || u == LS) break; } p++; continue; } break; } loc.linnum = 2; } } // generate a unique identifier for this string static Identifier idPool(in char[] str) { // StringValue sv; // uint len = s.length; // StringValue sv = stringtable.update(s, len); // Identifier* id = cast(Identifier*) sv.ptrvalue; // if( id is null ) if ((str in stringtable) == null) { stringtable[str] = new Identifier(str, TOK.TOKidentifier); } return stringtable[str]; } static void initKeywords() { // build character map cmtable_init(); // create keyword tokens & identifiers dparser.Keyword.initKeywords(); // create standard lexer tokens dparser.Token.createLexerTokens(); } // Combine two document comments into one. static char[] combineComments(char[] c1, char[] c2) { char[] c = c2; if (c1.length) { c = c1; if (c2.length) { c = c1 ~ "\n" ~ c2; } } return c; } // Decode UTF character. Issue error messages for invalid sequences. Return decoded character, advance p to last character in UTF sequence. //! fix uint decodeUTF() { ubyte *s = p; ubyte c = *s; assert(c & 0x80); if (!(c & 0x80)) return c; return cast(uint)'X'; /* dchar u; uint len; // Check length of remaining string up to 6 UTF-8 characters for( len = 1; len < 6 && s[len]; len++ ) { } /+ uint idx = 0; char* msg = utf_decodeChar( s, len, &idx, &u ); p += idx - 1; if( msg ) { error(msg); } +/ return u; */ } void error(...) { if ((mod !is null) && !global.gag) { writefln(formatLoc(loc, _arguments, _argptr)); /* char[] p = loc.toChars(); if( p.length ) writef( "%s: ", p ); writefx( stdout, _arguments, _argptr, 1 ); */ if (global.errors >= global.max_errors) // moderate blizzard of cascading messages throw new Exception("too many errors"); } global.errors++; } void errorLoc(Loc loc, ...) { if ((mod !is null) && !global.gag) { writefln(formatLoc(loc, _arguments, _argptr)); /* char[] p = loc.toChars(); if( p.length ) writef("%s: ", p); writefx(stdout, _arguments, _argptr, 1); */ if (global.errors >= 20) // moderate blizzard of cascading messages throw new Exception("too many errors"); } global.errors++; } TOK nextToken() { if (token.next) { Token *t = token.next; memcpy(&token, t, Token.sizeof); // t.next = freelist; // freelist = t; } else { scan(&token); } // token.print(); return token.value; } Token *peek(inout Token ct) { Token *t; if (ct.next) t = ct.next; else { t = new Token; scan(t); t.next = null; ct.next = t; } return t; } // Turn next token in buffer into a token. void scan(Token *t) { // debug writefln("scan token"); uint lastLine = loc.linnum; uint linnum; t.blockComment = null; t.lineComment = null; while (true) { t.ptr = p; // debug writefln( " p = %d, *p = ", cast(uint)p, cast(char)*p ); switch (*p) { case 0: case 0x1a: t.value = TOK.TOKeof; // end of file // debug writefln( " EOF" ); return; case ' ': case '\t': case '\v': case '\f': p++; // debug writefln( " whitespace" ); continue; // skip white space case '\r': // debug writefln( " cr" ); p++; if (*p != '\n') // if CR stands by itself loc.linnum++; continue; // skip white space case '\n': // debug writefln( " nl" ); p++; loc.linnum++; continue; // skip white space case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': t.value = number(t); return; /* #if CSTRINGS case '\'': t.value = charConstant(t, 0); return; case '"': t.value = stringConstant(t,0); return; case 'l': case 'L': if( p[1] == '\'') { p++; t.value = charConstant(t, 1); return; } else if( p[1] == '"') { p++; t.value = stringConstant(t, 1); return; } #else */ case '\'': // debug writefln( " char" ); t.value = charConstant(t, 0); return; case 'r': // debug writefln( " wysiwyg" ); if (p[1] != '"') goto case_identifier; p++; case '`': t.value = wysiwygStringConstant(t, *p); return; case 'x': // debug writefln( " hex string" ); if (p[1] != '"') goto case_identifier; p++; t.value = hexStringConstant(t); return; case '"': // debug writefln( " string" ); t.value = escapeStringConstant(t, 0); // debug writefln( t.ustring ); return; case '\\': // escaped string literal // debug writefln( " escaped string literal" ); uint c; stringbuffer.offset = 0; do { p++; c = escapeSequence(); stringbuffer.write(c); } while (*p == '\\'); // t.len = stringbuffer.offset; // stringbuffer.write(cast(byte)0); t.ustring = stringbuffer.toString; // memcpy( t.ustring.ptr, stringbuffer.data, stringbuffer.offset ); t.postfix = 0; t.value = TOK.TOKstring; return; case 'l': case 'L': // #endif case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'm': case 'n': case 'o': case 'p': case 'q': /*case 'r':*/ case 's': case 't': case 'u': case 'v': case 'w': /*case 'x':*/ case 'y': case 'z': case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case_identifier: { // debug writefln( " identifier" ); ubyte c; do { c = *++p; } while (isidchar(c) || (c & 0x80 && isUniAlpha(decodeUTF()))); // sv = stringtable.update((char *)t.ptr, p - t.ptr); char[] tmp; tmp.length = p - t.ptr; memcpy(tmp.ptr, t.ptr, p - t.ptr); Identifier id; Identifier *pid = tmp in stringtable; if (pid) { id = *pid; } if (id is null) { id = new Identifier(tmp, TOK.TOKidentifier); stringtable[tmp] = id; } t.identifier = id; t.value = cast(TOK)id.value; anyToken = 1; // if special identifier token if (*t.ptr == '_') { static char date[11 + 1]; static char time[8 + 1]; static char timestamp[24 + 1]; if (!date[0]) // lazy evaluation { //!! /+ time_t t; char *p; .time(&t); p = ctime(&t); assert(p); sprintf(date.ptr, "%.6s %.4s", p + 4, p + 20); sprintf(time.ptr, "%.8s", p + 11); sprintf(timestamp.ptr, "%.24s", p); +/ } if (mod && id is Id.FILE) { t.value = TOK.TOKstring; if (loc.filename.length) t.ustring = loc.filename; else t.ustring = mod.identifier.toChars(); goto Llen; } else if (mod && id == Id.LINE) { t.value = TOK.TOKint64v; t.uns64value = loc.linnum; } else if (id == Id.DATE) { t.value = TOK.TOKstring; //! t.ustring = date; goto Llen; } else if (id == Id.TIME) { t.value = TOK.TOKstring; //! t.ustring = time; goto Llen; } else if (id == Id.TIMESTAMP) { t.value = TOK.TOKstring; //! t.ustring = timestamp; Llen: t.postfix = 0; // t.len = strlen((char *)t.ustring); } } //printf("t.value = %d\n",t.value); return; } // comments case '/': p++; switch (*p) { case '=': p++; t.value = TOK.TOKdivass; return; case '*': // '/*' p++; linnum = loc.linnum; while (true) { while (true) { ubyte c = *p; switch (c) { case '/': break; case '\n': loc.linnum++; p++; continue; case '\r': p++; if (*p != '\n') loc.linnum++; continue; case 0: case 0x1A: error("unterminated /* */ comment"); p = end; t.value = TOK.TOKeof; return; default: if (c & 0x80) { uint u = decodeUTF(); if (u == PS || u == LS) loc.linnum++; } p++; continue; } break; } p++; if (p[-2] == '*' && p - 3 != t.ptr) break; } if (commentToken) { t.value = TOK.TOKcomment; return; } // if /** but not /**/ else if (doDocComment && t.ptr[2] == '*' && p - 4 != t.ptr) getDocComment(t, lastLine == linnum); //! ? continue; case '/': // do // style comments linnum = loc.linnum; while (1) { ubyte c = *++p; switch (c) { case '\n': break; case '\r': if (p[1] == '\n') p++; break; case 0: case 0x1a: if (commentToken) { p = end; t.value = TOK.TOKcomment; return; } if (doDocComment && t.ptr[2] == '/') getDocComment(t, lastLine == linnum); p = end; t.value = TOK.TOKeof; return; default: if (c & 0x80) { uint u = decodeUTF(); if (u == PS || u == LS) break; } continue; } break; } if (commentToken) { p++; loc.linnum++; t.value = TOK.TOKcomment; return; } if (doDocComment && t.ptr[2] == '/') getDocComment(t, lastLine == linnum); p++; loc.linnum++; continue; case '+': { int nest; linnum = loc.linnum; p++; nest = 1; while (1) { ubyte c = *p; switch (c) { case '/': p++; if (*p == '+') { p++; nest++; } continue; case '+': p++; if (*p == '/') { p++; if (--nest == 0) break; } continue; case '\r': p++; if (*p != '\n') loc.linnum++; continue; case '\n': loc.linnum++; p++; continue; case 0: case 0x1A: error("unterminated /+ +/ comment"); p = end; t.value = TOK.TOKeof; return; default: if (c & 0x80) { uint u = decodeUTF(); if (u == PS || u == LS) loc.linnum++; } p++; continue; } break; } if (commentToken) { t.value = TOK.TOKcomment; return; } if (doDocComment && t.ptr[2] == '+' && p - 4 != t.ptr) { // if /++ but not /++/ getDocComment(t, lastLine == linnum); } continue; } default: break; } t.value = TOK.TOKdiv; return; case '.': p++; if (isdigit(*p)) { p--; t.value = inreal(t); } else if (p[0] == '.') { if (p[1] == '.') { p += 2; t.value = TOK.TOKdotdotdot; } else { p++; t.value = TOK.TOKslice; } } else t.value = TOK.TOKdot; return; case '&': p++; if (*p == '=') { p++; t.value = TOK.TOKandass; } else if (*p == '&') { p++; t.value = TOK.TOKandand; } else t.value = TOK.TOKand; return; // |, ||, |= case '|': p++; if (*p == '=') { p++; t.value = TOK.TOKorass; } else if (*p == '|') { p++; t.value = TOK.TOKoror; } else t.value = TOK.TOKor; return; case '-': p++; if (*p == '=') { p++; t.value = TOK.TOKminass; } else if (*p == '-') { p++; t.value = TOK.TOKminusminus; } else t.value = TOK.TOKmin; return; // +, +=, ++ case '+': p++; if (*p == '=') { p++; t.value = TOK.TOKaddass; // += } else if (*p == '+') { p++; t.value = TOK.TOKplusplus; // ++ } else t.value = TOK.TOKadd; // + return; // <, <=, <<=, <<, <>=, <> case '<': p++; if (*p == '=') { p++; t.value = TOK.TOKle; // <= } else if (*p == '<') { p++; if (*p == '=') { p++; t.value = TOK.TOKshlass; // <<= } else t.value = TOK.TOKshl; // << } else if (*p == '>') { p++; if (*p == '=') { p++; t.value = TOK.TOKleg; // <>= } else t.value = TOK.TOKlg; // <> } else t.value = TOK.TOKlt; // < return; // >, >>, >>>, >=, >>=, >>>= case '>': p++; if (*p == '=') { p++; t.value = TOK.TOKge; // >= } else if (*p == '>') { p++; if (*p == '=') { p++; t.value = TOK.TOKshrass; // >>= } else if (*p == '>') { p++; if (*p == '=') { p++; t.value = TOK.TOKushrass; // >>>= } else t.value = TOK.TOKushr; // >>> } else t.value = TOK.TOKshr; // >> } else t.value = TOK.TOKgt; // > return; case '!': p++; if (*p == '=') { p++; if (*p == '=') { p++; t.value = TOK.TOKnotidentity; // !== } else t.value = TOK.TOKnotequal; // != } else if (*p == '<') { p++; if (*p == '>') { p++; if (*p == '=') { p++; t.value = TOK.TOKunord; // !<>= } else t.value = TOK.TOKue; // !<> } else if (*p == '=') { p++; t.value = TOK.TOKug; // !<= } else t.value = TOK.TOKuge; // !< } else if (*p == '>') { p++; if (*p == '=') { p++; t.value = TOK.TOKul; // !>= } else t.value = TOK.TOKule; // !> } else t.value = TOK.TOKnot; // ! return; case '=': p++; if (*p == '=') { p++; if (*p == '=') { p++; t.value = TOK.TOKidentity; // === } else t.value = TOK.TOKequal; // == } else t.value = TOK.TOKassign; // = return; case '~': p++; if (*p == '=') { p++; t.value = TOK.TOKcatass; // ~= } else t.value = TOK.TOKtilde; // ~ return; // SINGLE case '(': p++; t.value = TOK.TOKlparen; return; case ')': p++; t.value = TOK.TOKrparen; return; case '[': p++; t.value = TOK.TOKlbracket; return; case ']': p++; t.value = TOK.TOKrbracket; return; case '{': p++; t.value = TOK.TOKlcurly; return; case '}': p++; t.value = TOK.TOKrcurly; return; case '?': p++; t.value = TOK.TOKquestion; return; case ',': p++; t.value = TOK.TOKcomma; return; case ';': p++; t.value = TOK.TOKsemicolon; return; case ':': p++; t.value = TOK.TOKcolon; return; case '$': p++; t.value = TOK.TOKdollar; return; // DOUBLE case '*': p++; if (*p == '=') { p++; t.value = TOK.TOKmulass; } else t.value = TOK.TOKmul; return; case '%': p++; if (*p == '=') { p++; t.value = TOK.TOKmodass; } else t.value = TOK.TOKmod; return; case '^': p++; if (*p == '=') { p++; t.value = TOK.TOKxorass; } else t.value = TOK.TOKxor; return; // removed 148 case '~': p++; if( *p == '=' ) { p++; t.value = TOK.TOKcatass; } else t.value = TOK.TOKtilde; return; case '#': p++; Pragma(); continue; default: { debug writefln(" default char"); ubyte c = *p; if (c & 0x80) { uint u = decodeUTF(); // Check for start of unicode identifier if (isUniAlpha(u)) goto case_identifier; if (u == PS || u == LS) { loc.linnum++; p++; continue; } } if (isprint(c)) error("unsupported char '%s'", cast(char)c); else error("unsupported char 0x%02x", cast(ubyte)c); p++; continue; } } } } // Parse escape sequence. uint escapeSequence() { uint c; int n; int ndigits; c = *p; switch (c) { case '\'': case '"': case '?': case '\\': Lconsume: p++; break; case 'a': c = 7; goto Lconsume; case 'b': c = 8; goto Lconsume; case 'f': c = 12; goto Lconsume; case 'n': c = 10; goto Lconsume; case 'r': c = 13; goto Lconsume; case 't': c = 9; goto Lconsume; case 'v': c = 11; goto Lconsume; case 'u': ndigits = 4; goto Lhex; case 'U': ndigits = 8; goto Lhex; case 'x': ndigits = 2; Lhex: p++; c = *p; if (ishex(c)) { uint v; n = 0; v = 0; while (1) { if (isdigit(c)) c -= '0'; else if (islower(c)) c -= 'a' - 10; else c -= 'A' - 10; v = v * 16 + c; c = *++p; if (++n == ndigits) break; if (!ishex(c)) { error("escape hex sequence has %d hex digits instead of %d", n, ndigits); break; } } //! if( ndigits != 2 && !utf_isValidDchar(v)) //! error("invalid UTF character \\U%08x", v); c = v; } else error("undefined escape hex sequence \\%s\n", c); break; case '&': // named character entity for (ubyte *idstart = ++p; 1; p++) { switch (*p) { case ';': //!!! /+ c = HtmlNamedEntity(idstart, p - idstart); if( c == ~0 ) { error("unnamed character entity &%.*s;", p - idstart, idstart); c = ' '; } p++; +/ break; default: if (isalpha(*p) || (p != idstart + 1 && isdigit(*p))) continue; error("unterminated named entity"); break; } break; } break; case 0: case 0x1a: // end of file c = '\\'; break; default: if (isoctal(c)) { ubyte v; n = 0; do { v = v * 8 + (c - '0'); c = *++p; } while (++n < 3 && isoctal(c)); c = v; } else error("undefined escape sequence \\%s\n", c); break; } return c; } /************************************** */ TOK wysiwygStringConstant(Token *t, int tc) { uint c; Loc start = loc; p++; stringbuffer.offset = 0; while (1) { c = *p++; switch (c) { case '\n': loc.linnum++; break; case '\r': if (*p == '\n') continue; // ignore c = '\n'; // treat EndOfLine as \n character loc.linnum++; break; case 0: case 0x1a: error("unterminated string constant starting at %s", start.toChars()); t.ustring = ""; t.postfix = 0; return TOK.TOKstring; case '"': case '`': if (c == tc) { // t.len = stringbuffer.offset; stringbuffer.write(cast(byte)0); t.ustring = stringbuffer.toString; // t.ustring = (ubyte *)mem.malloc(stringbuffer.offset); // memcpy(t.ustring, stringbuffer.data, stringbuffer.offset); stringPostfix(t); return TOK.TOKstring; } break; default: if (c & 0x80) { p--; uint u = decodeUTF(); p++; if (u == PS || u == LS) loc.linnum++; stringbuffer.write(u); continue; } break; } stringbuffer.write(c); } } /************************************** * Lex hex strings: * x"0A ae 34FE BD" */ TOK hexStringConstant(Token *t) { uint c; Loc start = loc; uint n = 0; uint v; p++; stringbuffer.offset = 0; while (1) { c = *p++; switch (c) { case ' ': case '\t': case '\v': case '\f': continue; // skip white space case '\r': if (*p == '\n') continue; // ignore // Treat isolated '\r' as if it were a '\n' case '\n': loc.linnum++; continue; case 0: case 0x1a: error("unterminated string constant starting at %s", start.toChars()); t.ustring = ""; t.postfix = 0; return TOK.TOKstring; case '"': if (n & 1) { error("odd number (%d) of hex characters in hex string", n); stringbuffer.write(v); } // t.len = stringbuffer.offset; // stringbuffer.write(cast(byte)0); t.ustring = stringbuffer.toString; // t.ustring = (ubyte *)mem.malloc(stringbuffer.offset); // memcpy(t.ustring, stringbuffer.data, stringbuffer.offset); stringPostfix(t); return TOK.TOKstring; default: if (c >= '0' && c <= '9') c -= '0'; else if (c >= 'a' && c <= 'f') c -= 'a' - 10; else if (c >= 'A' && c <= 'F') c -= 'A' - 10; else if (c & 0x80) { p--; uint u = decodeUTF(); p++; if (u == PS || u == LS) loc.linnum++; else error("non-hex character \\u%x", u); } else error("non-hex character '%s'", c); if (n & 1) { v = (v << 4) | c; stringbuffer.write(v); } else v = c; n++; break; } } } /************************************** */ TOK escapeStringConstant(Token *t, int wide) { uint c; Loc start = loc; p++; stringbuffer.offset = 0; // debug writefln( "escape string constant: %s", std.string.toString( cast(char*)p ) ); while (1) { c = *p++; switch (c) { case '\\': switch (*p) { case 'u': case 'U': case '&': c = escapeSequence(); stringbuffer.write(c); continue; default: c = escapeSequence(); break; } break; case '\n': loc.linnum++; break; case '\r': if (*p == '\n') continue; // ignore c = '\n'; // treat EndOfLine as \n character loc.linnum++; break; case '"': // writefln( "end of string: ", stringbuffer.toString ); t.ustring = stringbuffer.toString().dup; // t.len = stringbuffer.offset; // stringbuffer.write(cast(byte)0); // t.ustring = (ubyte *)mem.malloc(stringbuffer.offset); // memcpy(t.ustring, stringbuffer.data, stringbuffer.offset); stringPostfix(t); return TOK.TOKstring; case 0: case 0x1a: p--; error("unterminated string constant starting at %s", start.toChars()); t.ustring = ""; // t.len = 0; t.postfix = 0; return TOK.TOKstring; default: if (c & 0x80) { p--; c = decodeUTF(); if (c == LS || c == PS) { c = '\n'; loc.linnum++; } p++; stringbuffer.write(cast(char)c); continue; } break; } stringbuffer.write(cast(char)c); // writefln( stringbuffer.toString ); } } //************************************** TOK charConstant(Token *t, int wide) { uint c; TOK tk = TOK.TOKcharv; //printf("Lexer.charConstant\n"); p++; c = *p++; switch (c) { case '\\': switch (*p) { case 'u': t.uns64value = escapeSequence(); tk = TOK.TOKwcharv; break; case 'U': case '&': t.uns64value = escapeSequence(); tk = TOK.TOKdcharv; break; default: t.uns64value = escapeSequence(); break; } break; case '\n': L1: loc.linnum++; case '\r': case 0: case 0x1a: case '\'': error("unterminated character constant"); return tk; default: if (c & 0x80) { p--; c = decodeUTF(); p++; if (c == LS || c == PS) goto L1; if (c < 0xd800 || (c >= 0xe000 && c < 0xfffe)) tk = TOK.TOKwcharv; else tk = TOK.TOKdcharv; } t.uns64value = c; break; } if (*p != '\'') { error("unterminated character constant"); return tk; } p++; return tk; } // Get postfix of string literal. void stringPostfix(Token *t) { switch (*p) { case 'c': case 'w': case 'd': t.postfix = *p; p++; break; default: t.postfix = 0; break; } } /*************************************** * Read \u or \U unicode sequence * Input: * u 'u' or 'U' */ /* uint Wchar(uint u) { uint value; uint n; ubyte c; uint nchars; nchars = (u == 'U') ? 8 : 4; value = 0; for (n = 0; 1; n++) { ++p; if( n == nchars) break; c = *p; if( !ishex(c)) { error("\\%s sequence must be followed by %d hex characters", u, nchars); break; } if( isdigit(c)) c -= '0'; else if( islower(c)) c -= 'a' - 10; else c -= 'A' - 10; value <<= 4; value |= c; } return value; } */ /************************************** * Read in a number. * If it's an integer, store it in tok.TKutok.Vlong. * integers can be decimal, octal or hex * Handle the suffixes U, UL, LU, L, etc. * If it's double, store it in tok.TKutok.Vdouble. * Returns: * TKnum * TKdouble,... */ TOK number(Token *t) { //debug writefln("Lexer.number()"); // We use a state machine to collect numbers enum STATE { STATE_initial, STATE_0, STATE_decimal, STATE_octal, STATE_octale, STATE_hex, STATE_binary, STATE_hex0, STATE_binary0, STATE_hexh, STATE_error } enum FLAGS { FLAGS_decimal = 1, // decimal FLAGS_unsigned = 2, // u or U suffix FLAGS_long = 4, // l or L suffix } FLAGS flags = FLAGS.FLAGS_decimal; int i; TOK result; int base; stringbuffer.offset = 0; // stringbuffer.data = null; STATE state = STATE.STATE_initial; ubyte *start = p; TOK _isreal() { p = start; return inreal(t); } while (true) { char c = cast(char)*p; switch (state) { case STATE.STATE_initial: // opening state if (c == '0') state = STATE.STATE_0; else state = STATE.STATE_decimal; break; case STATE.STATE_0: flags = cast(FLAGS)(flags & ~FLAGS.FLAGS_decimal); switch (c) { // #if ZEROH // case 'H': // 0h // case 'h': // goto hexh; // #endif case 'X': case 'x': state = STATE.STATE_hex0; break; case '.': if (p[1] == '.') // .. is a separate token goto done; case 'i': case 'f': case 'F': goto _Real; // #if ZEROH // case 'E': // case 'e': // goto case_hex; // #endif case 'B': case 'b': state = STATE.STATE_binary0; break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': state = STATE.STATE_octal; break; // #if ZEROH // case '8': case '9': case 'A': // case 'C': case 'D': case 'F': // case 'a': case 'c': case 'd': case 'f': // case_hex: // state = STATE.STATE_hexh; // break; // #endif case '_': state = STATE.STATE_octal; p++; continue; default: goto done; } break; case STATE.STATE_decimal: // reading decimal number // if its not a digit - decimal complete or not a decimal if (!isdigit(c)) { // debug writefln( "\tnon-digit( %s )", c ); // #if ZEROH // if( ishex(c) || c == 'H' || c == 'h' ) // goto hexh; // #endif //! wtf ? // ignore embedded _ if (c == '_') { p++; continue; } // check decimal point - make real if (c == '.' && p[1] != '.') goto _Real; // check for mantra - make real if (c == 'i' || c == 'f' || c == 'F' || c == 'e' || c == 'E') { _Real: // It's a real number. Back up and rescan as a real p = start; return inreal(t); } goto done; } break; case STATE.STATE_hex0: // reading hex number case STATE.STATE_hex: if (!ishex(c)) { if (c == '_') // ignore embedded _ { p++; continue; } if (c == '.' && p[1] != '.') goto _Real; if (c == 'P' || c == 'p' || c == 'i') goto _Real; if (state == STATE.STATE_hex0) error("Hex digit expected, not '%s'", c); goto done; } state = STATE.STATE_hex; break; // #if ZEROH // hexh: // state = STATE.STATE_hexh; // // case STATE.STATE_hexh: // parse numbers like 0FFh // if( !ishex(c)) // { // if( c == 'H' || c == 'h') // { // p++; // base = 16; // goto done; // } // else // { // // Check for something like 1E3 or 0E24 // if( memchr(stringbuffer.data.ptr, 'E', stringbuffer.offset) || memchr( stringbuffer.data.ptr, 'e', stringbuffer.offset)) // goto _Real; // error("Hex digit expected, not '%s'", c); // goto done; // } // } // break; // #endif case STATE.STATE_octal: // reading octal number case STATE.STATE_octale: // reading octal number with non-octal digits if (!isoctal(c)) { // #if ZEROH // if( ishex(c) || c == 'H' || c == 'h' ) // goto hexh; // #endif if (c == '_') // ignore embedded _ { p++; continue; } if (c == '.' && p[1] != '.') goto _Real; if (c == 'i') goto _Real; if (isdigit(c)) state = STATE.STATE_octale; else goto done; } break; case STATE.STATE_binary0: // starting binary number case STATE.STATE_binary: // reading binary number if (c != '0' && c != '1') { // #if ZEROH // if( ishex(c) || c == 'H' || c == 'h' ) // goto hexh; // #endif if (c == '_') // ignore embedded _ { p++; continue; } if (state == STATE.STATE_binary0) { error("binary digit expected"); state = STATE.STATE_error; break; } else goto done; } state = STATE.STATE_binary; break; case STATE.STATE_error: // for error recovery if (!isdigit(c)) // scan until non-digit goto done; break; default: assert(0); } stringbuffer.write(cast(char)c); p++; } done: stringbuffer.write(cast(char)0); // terminate string // debug writefln( "\tdigit complete( %s )", stringbuffer.toString ); if (state == STATE.STATE_octale) error("Octal digit expected"); uinteger_t n; // unsigned >=64 bit integer type if (stringbuffer.offset == 2 && (state == STATE.STATE_decimal || state == STATE.STATE_0)) n = stringbuffer.data[0] - '0'; else { // Convert string to integer char *p = cast(char *)stringbuffer.data.ptr; int r = 10; int d; if (*p == '0') { if (p[1] == 'x' || p[1] == 'X') { // "0x#" p += 2; r = 16; } else if (p[1] == 'b' || p[1] == 'B') { // "0b#" - binary p += 2; r = 2; } else if (isdigit(p[1])) { p += 1; r = 8; } } n = 0; while (true) { if (*p >= '0' && *p <= '9') d = *p - '0'; else if (*p >= 'a' && *p <= 'z') d = *p - 'a' + 10; else if (*p >= 'A' && *p <= 'Z') d = *p - 'A' + 10; else break; if (d >= r) break; if (n * r + d < n) { error("integer overflow"); break; } n = n * r + d; p++; } // if n needs more than 64 bits if (n.sizeof > 8 && n > 0xffffffffffffffffL) error("integer overflow"); } // Parse trailing 'u', 'U', 'l' or 'L' in any combination while (true) { ubyte f; switch (*p) { case 'U': case 'u': f = FLAGS.FLAGS_unsigned; goto L1; case 'L': case 'l': f = FLAGS.FLAGS_long; L1: p++; if (flags & f) error("unrecognized token"); flags = cast(FLAGS)(flags | f); continue; default: break; } break; } switch (flags) { case 0: /* Octal or Hexadecimal constant. * First that fits: int, uint, long, ulong */ if (n & 0x8000000000000000L) result = TOK.TOKuns64v; else if (n & 0xffffffff00000000L) result = TOK.TOKint64v; else if (n & 0x80000000) result = TOK.TOKuns32v; else result = TOK.TOKint32v; break; case FLAGS.FLAGS_decimal: /* First that fits: int, long, long long */ if (n & 0x8000000000000000L) { error("signed integer overflow"); result = TOK.TOKuns64v; } else if (n & 0xffffffff80000000L) result = TOK.TOKint64v; else result = TOK.TOKint32v; break; case FLAGS.FLAGS_unsigned: case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned: /* First that fits: uint, ulong */ if (n & 0xffffffff00000000L) result = TOK.TOKuns64v; else result = TOK.TOKuns32v; break; case FLAGS.FLAGS_decimal | FLAGS.FLAGS_long: if (n & 0x8000000000000000L) { error("signed integer overflow"); result = TOK.TOKuns64v; } else result = TOK.TOKint64v; break; case FLAGS.FLAGS_long: if (n & 0x8000000000000000L) result = TOK.TOKuns64v; else result = TOK.TOKint64v; break; case FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long: case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long: result = TOK.TOKuns64v; break; default: debug writefln("%x", flags); assert(0); } t.uns64value = n; return result; } /************************************** * Read in characters, converting them to real. * Bugs: * Exponent overflow not detected. * Too much requested precision is not detected. */ TOK inreal(Token *t) { int dblstate; uint c; char hex; // is this a hexadecimal-floating-constant? TOK result; //printf("Lexer.inreal()\n"); stringbuffer.offset = 0; dblstate = 0; hex = 0; Lnext: while (1) { // Get next char from input c = *p++; //printf("dblstate = %d, c = '%s'\n", dblstate, c); while (1) { switch (dblstate) { case 0: // opening state if (c == '0') dblstate = 9; else if (c == '.') dblstate = 3; else dblstate = 1; break; case 9: dblstate = 1; if (c == 'X' || c == 'x') { hex++; break; } case 1: // digits to left of . case 3: // digits to right of . case 7: // continuing exponent digits if (!isdigit(c) && !(hex && isxdigit(c))) { if (c == '_') goto Lnext; // ignore embedded '_' dblstate++; continue; } break; case 2: // no more digits to left of . if (c == '.') { dblstate++; break; } case 4: // no more digits to right of . if ((c == 'E' || c == 'e') || hex && (c == 'P' || c == 'p')) { dblstate = 5; hex = 0; // exponent is always decimal break; } if (hex) error("binary-exponent-part required"); goto done; case 5: // looking immediately to right of E dblstate++; if (c == '-' || c == '+') break; case 6: // 1st exponent digit expected if (!isdigit(c)) error("exponent expected"); dblstate++; break; case 8: // past end of exponent digits goto done; } break; } stringbuffer.write(c); } done: p--; stringbuffer.write(cast(byte)0); // #if _WIN32 && __DMC__ char *save = __locale_decpoint; __locale_decpoint = "."; // #endif t.float80value = strtold(cast(char *)stringbuffer.data.ptr, null); errno = 0; switch (*p) { case 'F': case 'f': strtof(cast(char *)stringbuffer.data.ptr, null); result = TOK.TOKfloat32v; p++; break; default: strtod(cast(char *)stringbuffer.data.ptr, null); result = TOK.TOKfloat64v; break; case 'L': case 'l': result = TOK.TOKfloat80v; p++; break; } if (*p == 'i' || *p == 'I') { p++; switch (result) { case TOK.TOKfloat32v: result = TOK.TOKimaginary32v; break; case TOK.TOKfloat64v: result = TOK.TOKimaginary64v; break; case TOK.TOKfloat80v: result = TOK.TOKimaginary80v; break; } } // #if _WIN32 && __DMC__ __locale_decpoint = save; // #endif if (errno == ERANGE) error("number is not representable"); return result; } /********************************************* * Do pragma. * Currently, the only pragma supported is: * #line linnum [filespec] */ void Pragma() { Token tok; int linnum; char[] filespec; Loc loc = this.loc; scan(&tok); if (tok.value != TOK.TOKidentifier || tok.identifier != Id.line) goto Lerr; scan(&tok); if (tok.value == TOK.TOKint32v || tok.value == TOK.TOKint64v) linnum = tok.uns64value - 1; else goto Lerr; while (1) { switch (*p) { case 0: case 0x1a: case '\n': Lnewline: this.loc.linnum = linnum; if (filespec.length) this.loc.filename = filespec; return; case '\r': p++; if (*p != '\n') { p--; goto Lnewline; } continue; case ' ': case '\t': case '\v': case '\f': p++; continue; // skip white space case '_': if (mod && memcmp(p, cast(char *)"__FILE__", 8) == 0) { p += 8; //! filespec = mem.strdup(loc.filename ? loc.filename : mod.identifier.toChars()); } continue; case '"': if (filespec) goto Lerr; stringbuffer.offset = 0; p++; while (1) { uint c; c = *p; switch (c) { case '\n': case '\r': case 0: case 0x1a: goto Lerr; case '"': stringbuffer.write(cast(byte)0); // filespec = mem.strdup((char *)stringbuffer.data); filespec = stringbuffer.toString.dup; p++; break; default: if (c & 0x80) { uint u = decodeUTF(); if (u == PS || u == LS) goto Lerr; } stringbuffer.write(c); p++; continue; } break; } continue; default: if (*p & 0x80) { uint u = decodeUTF(); if (u == PS || u == LS) goto Lnewline; } goto Lerr; } } Lerr: errorLoc(loc, "#line integer [\"filespec\"]\\n expected"); } /*************************************************** * Parse doc comment embedded between t.ptr and p. * Remove trailing blanks and tabs from lines. * Replace all newlines with \n. * Remove leading comment character from each line. * Decide if it's a lineComment or a blockComment. * Append to previous one for this token. */ void getDocComment(Token *t, uint lineComment) { auto OutBuffer buf = new OutBuffer; ubyte ct = t.ptr[2]; ubyte *q = t.ptr + 3; // start of comment text int linestart = 0; ubyte *qend = p; if (ct == '*' || ct == '+') qend -= 2; // Scan over initial row of ****'s or ++++'s or ////'s for (; q < qend; q++) { if (*q != ct) break; } // Remove trailing row of ****'s or ++++'s if (ct != '/') { for (; q < qend; qend--) { if (qend[-1] != ct) break; } } for (; q < qend; q++) { ubyte c = *q; switch (c) { case '*': case '+': if (linestart && c == ct) { linestart = 0; // Trim preceding whitespace up to preceding \n while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) buf.offset--; continue; } break; case ' ': case '\t': break; case '\r': if (q[1] == '\n') continue; // skip the \r goto Lnewline; default: if (c == 226) { // If LS or PS if (q[1] == 128 && (q[2] == 168 || q[2] == 169)) { q += 2; goto Lnewline; } } linestart = 0; break; Lnewline: c = '\n'; // replace all newlines with \n case '\n': linestart = 1; // Trim trailing whitespace while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) buf.offset--; break; } buf.write(c); } // Always end with a newline if (!buf.offset || buf.data[buf.offset - 1] != '\n') buf.writenl(); //buf.write(cast(char)0); // It's a line comment if the start of the doc comment comes // after other non-whitespace on the same line. // ubyte** dc = (lineComment && anyToken) // ? &t.lineComment // : &t.blockComment; char[] dc = (lineComment && anyToken) ? t.lineComment : t.blockComment; // Combine with previous doc comment, if any if (dc.length) dc = combineComments(dc, buf.toString().dup); else dc = buf.toString().dup; // writefln( dc ); if (lineComment && anyToken) t.lineComment = dc; else t.blockComment = dc; } } // character maps static ubyte[256] cmtable; const int CMoctal = 0x1; const int CMhex = 0x2; const int CMidchar = 0x4; ubyte isoctal(ubyte c) { return cmtable[c] & CMoctal; } ubyte ishex(ubyte c) { return cmtable[c] & CMhex; } ubyte isidchar(ubyte c) { return cmtable[c] & CMidchar; } static void cmtable_init() { for (uint c = 0; c < cmtable.length; c++) { if ('0' <= c && c <= '7') cmtable[c] |= CMoctal; if (isdigit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F')) cmtable[c] |= CMhex; if (isalnum(c) || c == '_') cmtable[c] |= CMidchar; } } /+ struct StringValue { union { int intvalue; void *ptrvalue; dchar *string; } char[] lstring; } #define CASE_BASIC_TYPES case TOKwchar: case TOKdchar: case TOKbit: case TOKbool: case TOKchar: case TOKint8: case TOKuns8: case TOKint16: case TOKuns16: case TOKint32: case TOKuns32: case TOKint64: case TOKuns64: case TOKfloat32: case TOKfloat64: case TOKfloat80: case TOKimaginary32: case TOKimaginary64: case TOKimaginary80: case TOKcomplex32: case TOKcomplex64: case TOKcomplex80: case TOKvoid: #define CASE_BASIC_TYPES_X(t) \ case TOKvoid: t = Type::tvoid; goto LabelX; \ case TOKint8: t = Type::tint8; goto LabelX; \ case TOKuns8: t = Type::tuns8; goto LabelX; \ case TOKint16: t = Type::tint16; goto LabelX; \ case TOKuns16: t = Type::tuns16; goto LabelX; \ case TOKint32: t = Type::tint32; goto LabelX; \ case TOKuns32: t = Type::tuns32; goto LabelX; \ case TOKint64: t = Type::tint64; goto LabelX; \ case TOKuns64: t = Type::tuns64; goto LabelX; \ case TOKfloat32: t = Type::tfloat32; goto LabelX; \ case TOKfloat64: t = Type::tfloat64; goto LabelX; \ case TOKfloat80: t = Type::tfloat80; goto LabelX; \ case TOKimaginary32: t = Type::timaginary32; goto LabelX; \ case TOKimaginary64: t = Type::timaginary64; goto LabelX; \ case TOKimaginary80: t = Type::timaginary80; goto LabelX; \ case TOKcomplex32: t = Type::tcomplex32; goto LabelX; \ case TOKcomplex64: t = Type::tcomplex64; goto LabelX; \ case TOKcomplex80: t = Type::tcomplex80; goto LabelX; \ case TOKbit: t = Type::tbit; goto LabelX; \ case TOKchar: t = Type::tchar; goto LabelX; \ case TOKwchar: t = Type::twchar; goto LabelX; \ case TOKdchar: t = Type::tdchar; goto LabelX; \ LabelX +/