/+
 *      Copyright (c) 1999-2006 by Digital Mars
 *      All Rights Reserved
 *      written by Walter Bright www.digitalmars.com
 *      License for redistribution is by either the Artistic License in artistic.txt, or the GNU General Public License in gnu.txt.
 *      See the included readme.txt for details.
 *      D Language conversion by: J Duncan
 +/

/**
 *      d language lexer
 */

module dparser.Lexer;

import dparser.Root;

import dparser.Tokens;
import dparser.Token;
import dparser.Keyword;

import dparser.Types;

import dparser.Module;
import dparser.Identifier;
import dparser.unialpha;

import dparser.OutBuffer;

//private import std.ctype;
//private import std.string;
//import dwf.core.debugapi;

int errno = 0;

//#if _WIN32 && __DMC__
// from \dm\src\include\setlocal.h
//extern "C" char * __cdecl __locale_decpoint;
char* __locale_decpoint;
//#endif
//const uint LS = 0x2028;	// UTF line separator
//const uint PS = 0x2029;	// UTF paragraph separator

//extern int isUniAlpha(unsigned u);
//extern int HtmlNamedEntity(unsigned char *p, int length);

/**
 *      Lexer object
 */

class Lexer
{
    static           Identifier[char[]]       stringtable;
    static OutBuffer stringbuffer;
    static Token     * freelist;

    Token            token;        // current token
    Module           mod;          // current module
    Loc              loc;          // for error messages
    ubyte            *base;        // pointer to start of buffer
    ubyte            *end;         // past end of buffer
    ubyte            *p;           // current character
    int              doDocComment; // collect doc comment information
    int              anyToken;     // !=0 means seen at least one token
    int              commentToken; // !=0 means comments are TOKcomment's


    this(Module mod, ubyte* base, uint begoffset, uint endoffset, int doDocComment, int commentToken)
    {
        if (stringbuffer is null) {
            stringbuffer = new OutBuffer;
        }
        loc = Loc(mod, 1);

        this.base         = base;
        this.end          = base + endoffset;
        this.p            = base + begoffset;
        this.mod          = mod;
        this.doDocComment = doDocComment;
        this.commentToken = commentToken;

        /*
         *      If first line starts with '#!', ignore the line
         */

        if (p[0] == '#' && p[1] == '!') {
            p += 2;
            while (true) {
                ubyte c = *p;
                switch (c) {
                case '\n':
                    p++;
                    break;

                case '\r':
                    p++;
                    if (*p == '\n') {
                        p++;
                    }
                    break;

                case 0:
                case 0x1A:
                    break;

                default:
                    if (c & 0x80) {
                        uint u = decodeUTF();
                        if (u == PS || u == LS) {
                            break;
                        }
                    }
                    p++;
                    continue;
                }
                break;
            }

            loc.linnum = 2;
        }
    }



    // generate a unique identifier for this string
    static Identifier idPool(in char[] str) {
//	    StringValue sv;
//	    uint len = s.length;
//	    StringValue sv = stringtable.update(s, len);
//	    Identifier* id = cast(Identifier*) sv.ptrvalue;
//	    if( id is null )
        if ((str in stringtable) == null) {
            stringtable[str] = new Identifier(str, TOK.TOKidentifier);
        }
        return(stringtable[str]);
    }

    static void initKeywords() {
        // build character map
        cmtable_init();

        // create keyword tokens & identifiers
        dparser.Keyword.initKeywords();

        // create standard lexer tokens
        dparser.Token.createLexerTokens();
    }

    // Combine two document comments into one.
    static char[] combineComments(char[] c1, char[] c2) {
        char[] c = c2;

        if (c1.length) {
            c = c1;
            if (c2.length) {
                c = c1 ~ "\n" ~ c2;
            }
        }
        return(c);
    }

    // Decode UTF character. Issue error messages for invalid sequences. Return decoded character, advance p to last character in UTF sequence.
    //! fix
    uint decodeUTF() {
        ubyte * s = p;
        ubyte c   = *s;

        assert(c & 0x80);
        if (!(c & 0x80)) {
            return(c);
        }

        return(cast(uint)'X');
        /*
         *  dchar u;
         *  uint len;
         *
         *
         *
         *  // Check length of remaining string up to 6 UTF-8 characters
         *  for( len = 1; len < 6 && s[len]; len++ )
         *  {
         *
         *  }
         *              /+
         *  uint idx = 0;
         *  char* msg = utf_decodeChar( s, len, &idx, &u );
         *  p += idx - 1;
         *  if( msg )
         *  {
         *              error(msg);
         *  }
         * +/
         *  return u;
         */
    }

    void error(...) {
        if ((mod !is null) && !global.gag) {
            writefln(formatLoc(loc, _arguments, _argptr));
            /*
             * char[] p = loc.toChars();
             * if( p.length )
             *  writef( "%s: ", p );
             * writefx( stdout, _arguments, _argptr, 1 );
             */
            if (global.errors >= global.max_errors) {                   // moderate blizzard of cascading messages
                throw new Exception("too many errors");
            }
        }

        global.errors++;
    }

    void errorLoc(Loc loc, ...) {
        if ((mod !is null) && !global.gag) {
            writefln(formatLoc(loc, _arguments, _argptr));
            /*
             * char[] p = loc.toChars();
             * if( p.length )
             *  writef("%s: ", p);
             * writefx(stdout, _arguments, _argptr, 1);
             */
            if (global.errors >= 20) {                  // moderate blizzard of cascading messages
                throw new Exception("too many errors");
            }
        }

        global.errors++;
    }


    TOK nextToken() {
        if (token.next) {
            Token* t = token.next;
            memcpy(&token, t, Token.sizeof);
//			t.next = freelist;
//			freelist = t;
        }
        else {
            scan(&token);
        }
//	    token.print();
        return(token.value);
    }

    Token* peek(inout Token ct) {
        Token* t;

        if (ct.next) {
            t = ct.next;
        }
        else {
            t = new Token;
            scan(t);
            t.next  = null;
            ct.next = t;
        }
        return(t);
    }

    // Turn next token in buffer into a token.

    void scan(Token* t) {
//		debug writefln("scan token");
        uint lastLine = loc.linnum;
        uint linnum;

        t.blockComment = null;
        t.lineComment  = null;
        while (true) {
            t.ptr = p;
//			debug writefln( "    p = %d, *p = ", cast(uint)p, cast(char)*p );
            switch (*p) {
            case 0:
            case 0x1a:
                t.value = TOK.TOKeof;                                           // end of file
//					debug writefln( "    EOF" );
                return;

            case ' ':
            case '\t':
            case '\v':
            case '\f':
                p++;
//					debug writefln( "    whitespace" );
                continue;                                                               // skip white space

            case '\r':
//					debug writefln( "    cr" );
                p++;
                if (*p != '\n') {                                               // if CR stands by itself
                    loc.linnum++;
                }
                continue;                                                               // skip white space

            case '\n':
//					debug writefln( "    nl" );
                p++;
                loc.linnum++;
                continue;                                                               // skip white space

            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                t.value = number(t);
                return;

/*
 * #if CSTRINGS
 *                          case '\'':
 *                              t.value = charConstant(t, 0);
 *                              return;
 *
 *                          case '"':
 *                              t.value = stringConstant(t,0);
 *                              return;
 *
 *                          case 'l':
 *                          case 'L':
 *                              if( p[1] == '\'')
 *                              {
 *                                  p++;
 *                                  t.value = charConstant(t, 1);
 *                                  return;
 *                              }
 *                              else if( p[1] == '"')
 *                              {
 *                                  p++;
 *                                  t.value = stringConstant(t, 1);
 *                                  return;
 *                              }
 * #else
 */
            case '\'':
//					debug writefln( "    char" );
                t.value = charConstant(t, 0);
                return;

            case 'r':
//					debug writefln( "    wysiwyg" );
                if (p[1] != '"') {
                    goto case_identifier;
                }
                p++;

            case '`':
                t.value = wysiwygStringConstant(t, *p);
                return;

            case 'x':
//					debug writefln( "    hex string" );
                if (p[1] != '"') {
                    goto case_identifier;
                }
                p++;
                t.value = hexStringConstant(t);
                return;


            case '"':
//					debug writefln( "    string" );
                t.value = escapeStringConstant(t, 0);
//					debug writefln( t.ustring );
                return;

            case '\\':                                  // escaped string literal
//					debug writefln( "    escaped string literal" );
                uint c;
                stringbuffer.offset = 0;
                do {
                    p++;
                    c = escapeSequence();
                    stringbuffer.write(c);
                } while (*p == '\\');
//					t.len = stringbuffer.offset;
//					stringbuffer.write(cast(byte)0);
                t.ustring = stringbuffer.toString;
//					memcpy( t.ustring.ptr, stringbuffer.data, stringbuffer.offset );
                t.postfix = 0;
                t.value   = TOK.TOKstring;
                return;

            case 'l':
            case 'L':
//	#endif

            case 'a':
            case 'b':
            case 'c':
            case 'd':
            case 'e':
            case 'f':
            case 'g':
            case 'h':
            case 'i':
            case 'j':
            case 'k':
            case 'm':
            case 'n':
            case 'o':
            case 'p':
            case 'q':                             /*case 'r':*/
            case 's':
            case 't':
            case 'u':
            case 'v':
            case 'w':                                         /*case 'x':*/
            case 'y':
            case 'z':
            case 'A':
            case 'B':
            case 'C':
            case 'D':
            case 'E':
            case 'F':
            case 'G':
            case 'H':
            case 'I':
            case 'J':
            case 'K':
            case 'M':
            case 'N':
            case 'O':
            case 'P':
            case 'Q':
            case 'R':
            case 'S':
            case 'T':
            case 'U':
            case 'V':
            case 'W':
            case 'X':
            case 'Y':
            case 'Z':
            case '_':
case_identifier:
                {
//					debug writefln( "    identifier" );
                    ubyte c;
                    do {
                        c = *++p;
                    } while (isidchar(c) || (c & 0x80 && isUniAlpha(decodeUTF())));

//					sv = stringtable.update((char *)t.ptr, p - t.ptr);
                    char[] tmp;
                    tmp.length = p - t.ptr;
                    memcpy(tmp.ptr, t.ptr, p - t.ptr);
                    Identifier id;
                    Identifier * pid = tmp in stringtable;
                    if (pid) {
                        id = *pid;
                    }

                    if (id is null) {
                        id               = new Identifier(tmp, TOK.TOKidentifier);
                        stringtable[tmp] = id;
                    }

                    t.identifier = id;
                    t.value      = cast(TOK)id.value;
                    anyToken     = 1;

                    // if special identifier token
                    if (*t.ptr == '_') {
                        static char date[11 + 1];
                        static char time[8 + 1];
                        static char timestamp[24 + 1];

                        if (!date[0]) {                         // lazy evaluation
                            //!!
                            /+
                             *      time_t t;
                             *      char *p;
                             *      .time(&t);
                             *      p = ctime(&t);
                             *      assert(p);
                             *      sprintf(date.ptr, "%.6s %.4s", p + 4, p + 20);
                             *      sprintf(time.ptr, "%.8s", p + 11);
                             *      sprintf(timestamp.ptr, "%.24s", p);
                             +/
                        }

                        if (mod && id is Id.FILE) {
                            t.value = TOK.TOKstring;
                            if (loc.filename.length) {
                                t.ustring = loc.filename;
                            }
                            else {
                                t.ustring = mod.identifier.toChars();
                            }
                            goto Llen;
                        }
                        else if (mod && id == Id.LINE) {
                            t.value      = TOK.TOKint64v;
                            t.uns64value = loc.linnum;
                        }
                        else if (id == Id.DATE) {
                            t.value = TOK.TOKstring;
                            //! t.ustring = date;
                            goto Llen;
                        }
                        else if (id == Id.TIME) {
                            t.value = TOK.TOKstring;
                            //! t.ustring = time;
                            goto Llen;
                        }
                        else if (id == Id.TIMESTAMP) {
                            t.value = TOK.TOKstring;
                            //! t.ustring = timestamp;
Llen:
                            t.postfix = 0;
//							t.len = strlen((char *)t.ustring);
                        }
                    }
                    //printf("t.value = %d\n",t.value);
                    return;
                }

            // comments
            case '/':
                p++;
                switch (*p) {
                case '=':
                    p++;
                    t.value = TOK.TOKdivass;
                    return;

                case '*':                               // '/*'
                    p++;
                    linnum = loc.linnum;
                    while (true) {
                        while (true) {
                            ubyte c = *p;
                            switch (c) {
                            case '/':
                                break;

                            case '\n':
                                loc.linnum++;
                                p++;
                                continue;

                            case '\r':
                                p++;
                                if (*p != '\n') {
                                    loc.linnum++;
                                }
                                continue;

                            case 0:
                            case 0x1A:
                                error("unterminated /* */ comment");
                                p       = end;
                                t.value = TOK.TOKeof;
                                return;

                            default:
                                if (c & 0x80) {
                                    uint u = decodeUTF();
                                    if (u == PS || u == LS) {
                                        loc.linnum++;
                                    }
                                }
                                p++;
                                continue;
                            }
                            break;
                        }
                        p++;
                        if (p[-2] == '*' && p - 3 != t.ptr) {
                            break;
                        }
                    }

                    if (commentToken) {
                        t.value = TOK.TOKcomment;
                        return;
                    }
                    // if /** but not /**/
                    else if (doDocComment && t.ptr[2] == '*' && p - 4 != t.ptr) {
                        getDocComment(t, lastLine == linnum);                                           //! ?
                    }
                    continue;

                case '/':                                       // do // style comments
                    linnum = loc.linnum;
                    while (1) {
                        ubyte c = *++p;
                        switch (c) {
                        case '\n':
                            break;

                        case '\r':
                            if (p[1] == '\n') {
                                p++;
                            }
                            break;

                        case 0:
                        case 0x1a:
                            if (commentToken) {
                                p       = end;
                                t.value = TOK.TOKcomment;
                                return;
                            }
                            if (doDocComment && t.ptr[2] == '/') {
                                getDocComment(t, lastLine == linnum);
                            }
                            p       = end;
                            t.value = TOK.TOKeof;
                            return;

                        default:
                            if (c & 0x80) {
                                uint u = decodeUTF();
                                if (u == PS || u == LS) {
                                    break;
                                }
                            }
                            continue;
                        }
                        break;
                    }

                    if (commentToken) {
                        p++;
                        loc.linnum++;
                        t.value = TOK.TOKcomment;
                        return;
                    }
                    if (doDocComment && t.ptr[2] == '/') {
                        getDocComment(t, lastLine == linnum);
                    }

                    p++;
                    loc.linnum++;
                    continue;

                case '+':
                    {
                        int nest;
                        linnum = loc.linnum;
                        p++;
                        nest = 1;
                        while (1) {
                            ubyte c = *p;
                            switch (c) {
                            case '/':
                                p++;
                                if (*p == '+') {
                                    p++;
                                    nest++;
                                }
                                continue;

                            case '+':
                                p++;
                                if (*p == '/') {
                                    p++;
                                    if (--nest == 0) {
                                        break;
                                    }
                                }
                                continue;

                            case '\r':
                                p++;
                                if (*p != '\n') {
                                    loc.linnum++;
                                }
                                continue;

                            case '\n':
                                loc.linnum++;
                                p++;
                                continue;

                            case 0:
                            case 0x1A:
                                error("unterminated /+ +/ comment");
                                p       = end;
                                t.value = TOK.TOKeof;
                                return;

                            default:
                                if (c & 0x80) {
                                    uint u = decodeUTF();
                                    if (u == PS || u == LS) {
                                        loc.linnum++;
                                    }
                                }
                                p++;
                                continue;
                            }
                            break;
                        }
                        if (commentToken) {
                            t.value = TOK.TOKcomment;
                            return;
                        }
                        if (doDocComment && t.ptr[2] == '+' && p - 4 != t.ptr) {
                            // if /++ but not /++/
                            getDocComment(t, lastLine == linnum);
                        }
                        continue;
                    }

                default:
                    break;
                }
                t.value = TOK.TOKdiv;
                return;

            case '.':
                p++;
                if (isdigit(*p)) {
                    p--;
                    t.value = inreal(t);
                }
                else if (p[0] == '.') {
                    if (p[1] == '.') {
                        p      += 2;
                        t.value = TOK.TOKdotdotdot;
                    }
                    else {
                        p++;
                        t.value = TOK.TOKslice;
                    }
                }
                else {
                    t.value = TOK.TOKdot;
                }
                return;

            case '&':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKandass;
                }
                else if (*p == '&') {
                    p++;
                    t.value = TOK.TOKandand;
                }
                else {
                    t.value = TOK.TOKand;
                }
                return;

            // |, ||, |=
            case '|':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKorass;
                }
                else if (*p == '|') {
                    p++;
                    t.value = TOK.TOKoror;
                }
                else {
                    t.value = TOK.TOKor;
                }
                return;

            case '-':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKminass;
                }
                else if (*p == '-') {
                    p++;
                    t.value = TOK.TOKminusminus;
                }
                else {
                    t.value = TOK.TOKmin;
                }
                return;

            // +, +=, ++
            case '+':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKaddass;                                    // +=
                }
                else if (*p == '+') {
                    p++;
                    t.value = TOK.TOKplusplus;                                          // ++
                }
                else {
                    t.value = TOK.TOKadd;                                                       // +
                }
                return;

            // <, <=, <<=, <<, <>=, <>
            case '<':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKle;                                                // <=
                }
                else if (*p == '<') {
                    p++;
                    if (*p == '=') {
                        p++;
                        t.value = TOK.TOKshlass;                                        // <<=
                    }
                    else {
                        t.value = TOK.TOKshl;                                           // <<
                    }
                }
                else if (*p == '>') {
                    p++;
                    if (*p == '=') {
                        p++;
                        t.value = TOK.TOKleg;                                           // <>=
                    }
                    else {
                        t.value = TOK.TOKlg;                                            // <>
                    }
                }
                else {
                    t.value = TOK.TOKlt;                                                // <
                }
                return;

            // >, >>, >>>, >=, >>=, >>>=
            case '>':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKge;                                                // >=
                }
                else if (*p == '>') {
                    p++;
                    if (*p == '=') {
                        p++;
                        t.value = TOK.TOKshrass;                                        // >>=
                    }
                    else if (*p == '>') {
                        p++;
                        if (*p == '=') {
                            p++;
                            t.value = TOK.TOKushrass;                           // >>>=
                        }
                        else {
                            t.value = TOK.TOKushr;                                      // >>>
                        }
                    }
                    else {
                        t.value = TOK.TOKshr;                                           // >>
                    }
                }
                else {
                    t.value = TOK.TOKgt;                                                // >
                }
                return;

            case '!':
                p++;
                if (*p == '=') {
                    p++;
                    if (*p == '=') {
                        p++;
                        t.value = TOK.TOKnotidentity;                                   // !==
                    }
                    else {
                        t.value = TOK.TOKnotequal;                                      // !=
                    }
                }
                else if (*p == '<') {
                    p++;
                    if (*p == '>') {
                        p++;
                        if (*p == '=') {
                            p++;
                            t.value = TOK.TOKunord;                             // !<>=
                        }
                        else {
                            t.value = TOK.TOKue;                                // !<>
                        }
                    }
                    else if (*p == '=') {
                        p++;
                        t.value = TOK.TOKug;                                    // !<=
                    }
                    else {
                        t.value = TOK.TOKuge;                                   // !<
                    }
                }
                else if (*p == '>') {
                    p++;
                    if (*p == '=') {
                        p++;
                        t.value = TOK.TOKul;                                    // !>=
                    }
                    else {
                        t.value = TOK.TOKule;                                   // !>
                    }
                }
                else {
                    t.value = TOK.TOKnot;                                       // !
                }
                return;

            case '=':
                p++;
                if (*p == '=') {
                    p++;
                    if (*p == '=') {
                        p++;
                        t.value = TOK.TOKidentity;                                      // ===
                    }
                    else {
                        t.value = TOK.TOKequal;                                         // ==
                    }
                }
                else {
                    t.value = TOK.TOKassign;                                    // =
                }
                return;

            case '~':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKcatass;                                            // ~=
                }
                else {
                    t.value = TOK.TOKtilde;                                             // ~
                }
                return;

            // SINGLE
            case '(': p++; t.value = TOK.TOKlparen;     return;

            case ')': p++; t.value = TOK.TOKrparen;     return;

            case '[': p++; t.value = TOK.TOKlbracket;   return;

            case ']': p++; t.value = TOK.TOKrbracket;   return;

            case '{': p++; t.value = TOK.TOKlcurly;     return;

            case '}': p++; t.value = TOK.TOKrcurly;     return;

            case '?': p++; t.value = TOK.TOKquestion;   return;

            case ',': p++; t.value = TOK.TOKcomma;              return;

            case ';': p++; t.value = TOK.TOKsemicolon;  return;

            case ':': p++; t.value = TOK.TOKcolon;              return;

            case '$': p++; t.value = TOK.TOKdollar;     return;

            // DOUBLE
            case '*': p++; if (*p == '=') {
                    p++; t.value = TOK.TOKmulass;
            }
                else {
                    t.value = TOK.TOKmul;
                } return;

            case '%': p++; if (*p == '=') {
                    p++; t.value = TOK.TOKmodass;
            }
                else {
                    t.value = TOK.TOKmod;
                } return;

            case '^': p++; if (*p == '=') {
                    p++; t.value = TOK.TOKxorass;
            }
                else {
                    t.value = TOK.TOKxor;
                } return;

// removed 148	case '~': p++; if( *p == '=' ) { p++; t.value = TOK.TOKcatass; } else t.value = TOK.TOKtilde; return;


            case '#':
                p++;
                Pragma();
                continue;

            default:
                {
                    debug writefln("    default char");
                    ubyte c = *p;
                    if (c & 0x80) {
                        uint u = decodeUTF();
                        // Check for start of unicode identifier
                        if (isUniAlpha(u)) {
                            goto case_identifier;
                        }

                        if (u == PS || u == LS) {
                            loc.linnum++;
                            p++;
                            continue;
                        }
                    }
                    if (isprint(c)) {
                        error("unsupported char '%s'", cast(char)c);
                    }
                    else {
                        error("unsupported char 0x%02x", cast(ubyte)c);
                    }
                    p++;
                    continue;
                }
            }
        }
    }



    // Parse escape sequence.
    uint escapeSequence() {
        uint c;
        int  n;
        int  ndigits;

        c = *p;
        switch (c) {
        case '\'':
        case '"':
        case '?':
        case '\\':
Lconsume:
            p++;
            break;

        case 'a':       c = 7;          goto Lconsume;

        case 'b':       c = 8;          goto Lconsume;

        case 'f':       c = 12;         goto Lconsume;

        case 'n':       c = 10;         goto Lconsume;

        case 'r':       c = 13;         goto Lconsume;

        case 't':       c = 9;          goto Lconsume;

        case 'v':       c = 11;         goto Lconsume;

        case 'u':
            ndigits = 4;
            goto Lhex;

        case 'U':
            ndigits = 8;
            goto Lhex;

        case 'x':
            ndigits = 2;
Lhex:
            p++;
            c = *p;
            if (ishex(c)) {
                uint v;
                n = 0;
                v = 0;
                while (1) {
                    if (isdigit(c)) {
                        c -= '0';
                    }
                    else if (islower(c)) {
                        c -= 'a' - 10;
                    }
                    else {
                        c -= 'A' - 10;
                    }
                    v = v * 16 + c;
                    c = *++p;
                    if (++n == ndigits) {
                        break;
                    }
                    if (!ishex(c)) {
                        error("escape hex sequence has %d hex digits instead of %d", n, ndigits);
                        break;
                    }
                }
//!				    if( ndigits != 2 && !utf_isValidDchar(v))
//!						error("invalid UTF character \\U%08x", v);
                c = v;
            }
            else {
                error("undefined escape hex sequence \\%s\n", c);
            }
            break;

        case '&':                                       // named character entity
            for (ubyte *idstart = ++p; 1; p++) {
                switch (*p) {
                case ';':
                    //!!!
                    /+
                     * c = HtmlNamedEntity(idstart, p - idstart);
                     * if( c == ~0 )
                     * {
                     *      error("unnamed character entity &%.*s;", p - idstart, idstart);
                     *      c = ' ';
                     * }
                     *
                     * p++;
                     +/
                    break;

                default:
                    if (isalpha(*p) || (p != idstart + 1 && isdigit(*p))) {
                        continue;
                    }
                    error("unterminated named entity");
                    break;
                }
                break;
            }
            break;

        case 0:
        case 0x1a:                                      // end of file
            c = '\\';
            break;

        default:
            if (isoctal(c)) {
                ubyte v;
                n = 0;
                do {
                    v = v * 8 + (c - '0');
                    c = *++p;
                } while (++n < 3 && isoctal(c));
                c = v;
            }
            else {
                error("undefined escape sequence \\%s\n", c);
            }
            break;
        }
        return(c);
    }

    /**************************************
     */

    TOK wysiwygStringConstant(Token *t, int tc) {
        uint c;
        Loc  start = loc;

        p++;
        stringbuffer.offset = 0;
        while (1) {
            c = *p++;
            switch (c) {
            case '\n':
                loc.linnum++;
                break;

            case '\r':
                if (*p == '\n') {
                    continue;                           // ignore
                }
                c = '\n';                               // treat EndOfLine as \n character
                loc.linnum++;
                break;

            case 0:
            case 0x1a:
                error("unterminated string constant starting at %s", start.toChars());
                t.ustring = "";
                t.postfix = 0;
                return(TOK.TOKstring);

            case '"':
            case '`':
                if (c == tc) {
//				    t.len = stringbuffer.offset;
                    stringbuffer.write(cast(byte)0);
                    t.ustring = stringbuffer.toString;
//				    t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);
//				    memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);
                    stringPostfix(t);
                    return(TOK.TOKstring);
                }
                break;

            default:
                if (c & 0x80) {
                    p--;
                    uint u = decodeUTF();
                    p++;
                    if (u == PS || u == LS) {
                        loc.linnum++;
                    }
                    stringbuffer.write(u);
                    continue;
                }
                break;
            }
            stringbuffer.write(c);
        }
    }

    /**************************************
     * Lex hex strings:
     *	x"0A ae 34FE BD"
     */

    TOK hexStringConstant(Token *t) {
        uint c;
        Loc  start = loc;
        uint n     = 0;
        uint v;

        p++;
        stringbuffer.offset = 0;
        while (1) {
            c = *p++;
            switch (c) {
            case ' ':
            case '\t':
            case '\v':
            case '\f':
                continue;                                               // skip white space

            case '\r':
                if (*p == '\n') {
                    continue;                                           // ignore
                }

            // Treat isolated '\r' as if it were a '\n'
            case '\n':
                loc.linnum++;
                continue;

            case 0:
            case 0x1a:
                error("unterminated string constant starting at %s", start.toChars());
                t.ustring = "";
                t.postfix = 0;
                return(TOK.TOKstring);

            case '"':
                if (n & 1) {
                    error("odd number (%d) of hex characters in hex string", n);
                    stringbuffer.write(v);
                }
//				t.len = stringbuffer.offset;
//				stringbuffer.write(cast(byte)0);
                t.ustring = stringbuffer.toString;
//				t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);
//				memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);
                stringPostfix(t);
                return(TOK.TOKstring);

            default:
                if (c >= '0' && c <= '9') {
                    c -= '0';
                }
                else if (c >= 'a' && c <= 'f') {
                    c -= 'a' - 10;
                }
                else if (c >= 'A' && c <= 'F') {
                    c -= 'A' - 10;
                }
                else if (c & 0x80) {
                    p--;
                    uint u = decodeUTF();
                    p++;
                    if (u == PS || u == LS) {
                        loc.linnum++;
                    }
                    else {
                        error("non-hex character \\u%x", u);
                    }
                }
                else {
                    error("non-hex character '%s'", c);
                }
                if (n & 1) {
                    v = (v << 4) | c;
                    stringbuffer.write(v);
                }
                else {
                    v = c;
                }
                n++;
                break;
            }
        }
    }

    /**************************************
     */

    TOK escapeStringConstant(Token *t, int wide) {
        uint c;
        Loc  start = loc;

        p++;
        stringbuffer.offset = 0;
        //    debug writefln( "escape string constant: %s", std.string.toString( cast(char*)p ) );
        while (1) {
            c = *p++;
            switch (c) {
            case '\\':
                switch (*p) {
                case 'u':
                case 'U':
                case '&':
                    c = escapeSequence();
                    stringbuffer.write(c);
                    continue;

                default:
                    c = escapeSequence();
                    break;
                }
                break;

            case '\n':
                loc.linnum++;
                break;

            case '\r':
                if (*p == '\n') {
                    continue;                           // ignore
                }
                c = '\n';                               // treat EndOfLine as \n character
                loc.linnum++;
                break;

            case '"':
//			        writefln( "end of string: ", stringbuffer.toString );
                t.ustring = stringbuffer.toString().dup;
                //				t.len = stringbuffer.offset;
                //				stringbuffer.write(cast(byte)0);
                //				t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);
                //				memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);
                stringPostfix(t);

                return(TOK.TOKstring);

            case 0:
            case 0x1a:
                p--;
                error("unterminated string constant starting at %s", start.toChars());
                t.ustring = "";
//					t.len = 0;
                t.postfix = 0;
                return(TOK.TOKstring);

            default:
                if (c & 0x80) {
                    p--;
                    c = decodeUTF();
                    if (c == LS || c == PS) {
                        c = '\n';
                        loc.linnum++;
                    }
                    p++;
                    stringbuffer.write(cast(char)c);
                    continue;
                }
                break;
            }
            stringbuffer.write(cast(char)c);
//			writefln( stringbuffer.toString );
        }
    }

    //**************************************
    TOK charConstant(Token *t, int wide) {
        uint c;
        TOK  tk = TOK.TOKcharv;

        //printf("Lexer.charConstant\n");
        p++;
        c = *p++;
        switch (c) {
        case '\\':
            switch (*p) {
            case 'u':
                t.uns64value = escapeSequence();
                tk           = TOK.TOKwcharv;
                break;

            case 'U':
            case '&':
                t.uns64value = escapeSequence();
                tk           = TOK.TOKdcharv;
                break;

            default:
                t.uns64value = escapeSequence();
                break;
            }
            break;

        case '\n':
L1:
            loc.linnum++;

        case '\r':
        case 0:
        case 0x1a:
        case '\'':
            error("unterminated character constant");
            return(tk);

        default:
            if (c & 0x80) {
                p--;
                c = decodeUTF();
                p++;
                if (c == LS || c == PS) {
                    goto L1;
                }
                if (c < 0xd800 || (c >= 0xe000 && c < 0xfffe)) {
                    tk = TOK.TOKwcharv;
                }
                else {
                    tk = TOK.TOKdcharv;
                }
            }
            t.uns64value = c;
            break;
        }

        if (*p != '\'') {
            error("unterminated character constant");
            return(tk);
        }
        p++;
        return(tk);
    }

    // Get postfix of string literal.
    void stringPostfix(Token *t) {
        switch (*p) {
        case 'c':
        case 'w':
        case 'd':
            t.postfix = *p;
            p++;
            break;

        default:
            t.postfix = 0;
            break;
        }
    }

    /***************************************
     * Read \u or \U unicode sequence
     * Input:
     *	u	'u' or 'U'
     */
    /*
     * uint Wchar(uint u)
     * {
     *  uint value;
     *  uint n;
     *  ubyte c;
     *  uint nchars;
     *
     *  nchars = (u == 'U') ? 8 : 4;
     *  value = 0;
     *  for (n = 0; 1; n++)
     *  {
     * ++p;
     *              if( n == nchars)
     *                  break;
     *              c = *p;
     *              if( !ishex(c))
     *              {
     *                      error("\\%s sequence must be followed by %d hex characters", u, nchars);
     *                  break;
     *              }
     *              if( isdigit(c))
     *                  c -= '0';
     *              else if( islower(c))
     *                  c -= 'a' - 10;
     *              else
     *                  c -= 'A' - 10;
     *              value <<= 4;
     *              value |= c;
     *  }
     *  return value;
     * }
     */

    /**************************************
     * Read in a number.
     * If it's an integer, store it in tok.TKutok.Vlong.
     *	integers can be decimal, octal or hex
     *	Handle the suffixes U, UL, LU, L, etc.
     * If it's double, store it in tok.TKutok.Vdouble.
     * Returns:
     *	TKnum
     *	TKdouble,...
     */

    TOK number(Token *t) {
        //debug writefln("Lexer.number()");
        // We use a state machine to collect numbers
        enum STATE {
            STATE_initial,
            STATE_0,
            STATE_decimal,
            STATE_octal,
            STATE_octale,
            STATE_hex,
            STATE_binary,
            STATE_hex0,
            STATE_binary0,
            STATE_hexh,
            STATE_error
        }

        enum FLAGS {
            FLAGS_decimal  = 1,                         // decimal
            FLAGS_unsigned = 2,                         // u or U suffix
            FLAGS_long     = 4,                         // l or L suffix
        }
        FLAGS flags = FLAGS.FLAGS_decimal;

        int   i;
        TOK   result;
        int   base;

        stringbuffer.offset = 0;
//		stringbuffer.data = null;
        STATE state   = STATE.STATE_initial;
        ubyte * start = p;

        TOK _isreal() {
            p = start;
            return(inreal(t));
        }

        while (true) {
            char c = cast(char)*p;
            switch (state) {
            case STATE.STATE_initial:                           // opening state
                if (c == '0') {
                    state = STATE.STATE_0;
                }
                else {
                    state = STATE.STATE_decimal;
                }
                break;

            case STATE.STATE_0:
                flags = cast(FLAGS)(flags & ~FLAGS.FLAGS_decimal);
                switch (c) {
                //	#if ZEROH
//					    case 'H':			// 0h
//					    case 'h':
//							goto hexh;
                //	#endif
                case 'X':
                case 'x':
                    state = STATE.STATE_hex0;
                    break;

                case '.':
                    if (p[1] == '.') {                                          // .. is a separate token
                        goto done;
                    }

                case 'i':
                case 'f':
                case 'F':
                    goto _Real;

                //	#if ZEROH
//					    case 'E':
//					    case 'e':
//							goto case_hex;
                //	#endif
                case 'B':
                case 'b':
                    state = STATE.STATE_binary0;
                    break;

                case '0':
                case '1':
                case '2':
                case '3':
                case '4':
                case '5':
                case '6':
                case '7':
                    state = STATE.STATE_octal;
                    break;

                //	#if ZEROH
//					    case '8': case '9': case 'A':
//					    case 'C': case 'D': case 'F':
//					    case 'a': case 'c': case 'd': case 'f':
//						    case_hex:
//							state = STATE.STATE_hexh;
//							break;
                //	#endif
                case '_':
                    state = STATE.STATE_octal;
                    p++;
                    continue;

                default:
                    goto done;
                }
                break;

            case STATE.STATE_decimal:                           // reading decimal number

                // if its not a digit - decimal complete or not a decimal
                if (!isdigit(c)) {
//						debug writefln( "\tnon-digit( %s )", c );
                    //	#if ZEROH
//					    if( ishex(c) || c == 'H' || c == 'h' )
//							goto hexh;
                    //	#endif
                    //! wtf ?
                    // ignore embedded _
                    if (c == '_') {
                        p++;
                        continue;
                    }

                    // check decimal point - make real
                    if (c == '.' && p[1] != '.') {
                        goto _Real;
                    }

                    // check for mantra - make real
                    if (c == 'i' || c == 'f' || c == 'F' || c == 'e' || c == 'E') {
_Real:                  // It's a real number. Back up and rescan as a real
                        p = start;
                        return(inreal(t));
                    }

                    goto done;
                }
                break;

            case STATE.STATE_hex0:                              // reading hex number
            case STATE.STATE_hex:
                if (!ishex(c)) {
                    if (c == '_') {                                     // ignore embedded _
                        p++;
                        continue;
                    }
                    if (c == '.' && p[1] != '.') {
                        goto _Real;
                    }
                    if (c == 'P' || c == 'p' || c == 'i') {
                        goto _Real;
                    }
                    if (state == STATE.STATE_hex0) {
                        error("Hex digit expected, not '%s'", c);
                    }
                    goto done;
                }
                state = STATE.STATE_hex;
                break;

            //	#if ZEROH
//				    hexh:
//						state = STATE.STATE_hexh;
//
//				    case STATE.STATE_hexh:		// parse numbers like 0FFh
//						if( !ishex(c))
//						{
//						    if( c == 'H' || c == 'h')
//						    {
//								p++;
//								base = 16;
//								goto done;
//						    }
//						    else
//						    {
//								// Check for something like 1E3 or 0E24
//								if( memchr(stringbuffer.data.ptr, 'E', stringbuffer.offset) || memchr( stringbuffer.data.ptr, 'e', stringbuffer.offset))
//								    goto _Real;
//								error("Hex digit expected, not '%s'", c);
//								goto done;
//						    }
//						}
//						break;
            //		#endif

            case STATE.STATE_octal:                                     // reading octal number
            case STATE.STATE_octale:                                    // reading octal number with non-octal digits
                if (!isoctal(c)) {
//				#if ZEROH
//						    if( ishex(c) || c == 'H' || c == 'h' )
//								goto hexh;
//				#endif
                    if (c == '_') {                                             // ignore embedded _
                        p++;
                        continue;
                    }
                    if (c == '.' && p[1] != '.') {
                        goto _Real;
                    }
                    if (c == 'i') {
                        goto _Real;
                    }
                    if (isdigit(c)) {
                        state = STATE.STATE_octale;
                    }
                    else {
                        goto done;
                    }
                }
                break;

            case STATE.STATE_binary0:                                   // starting binary number
            case STATE.STATE_binary:                                    // reading binary number
                if (c != '0' && c != '1') {
                    //			#if ZEROH
//						    if( ishex(c) || c == 'H' || c == 'h' )
//								goto hexh;
                    //			#endif
                    if (c == '_') {                                             // ignore embedded _
                        p++;
                        continue;
                    }
                    if (state == STATE.STATE_binary0) {
                        error("binary digit expected");
                        state = STATE.STATE_error;
                        break;
                    }
                    else {
                        goto done;
                    }
                }
                state = STATE.STATE_binary;
                break;

            case STATE.STATE_error:                                     // for error recovery
                if (!isdigit(c)) {                                      // scan until non-digit
                    goto done;
                }
                break;

            default:
                assert(0);
            }
            stringbuffer.write(cast(char)c);
            p++;
        }
done:
        stringbuffer.write(cast(char)0);                // terminate string

//		debug writefln( "\tdigit complete( %s )", stringbuffer.toString );

        if (state == STATE.STATE_octale) {
            error("Octal digit expected");
        }

        uinteger_t n;                           // unsigned >=64 bit integer type

        if (stringbuffer.offset == 2 && (state == STATE.STATE_decimal || state == STATE.STATE_0)) {
            n = stringbuffer.data[0] - '0';
        }
        else {
            // Convert string to integer
            char* p = cast(char*)stringbuffer.data.ptr;
            int r   = 10;
            int d;
            if (*p == '0') {
                if (p[1] == 'x' || p[1] == 'X') {
                    // "0x#"
                    p += 2;
                    r  = 16;
                }
                else if (p[1] == 'b' || p[1] == 'B') {
                    // "0b#" - binary
                    p += 2;
                    r  = 2;
                }
                else if (isdigit(p[1])) {
                    p += 1;
                    r  = 8;
                }
            }

            n = 0;

            while (true) {
                if (*p >= '0' && *p <= '9') {
                    d = *p - '0';
                }
                else if (*p >= 'a' && *p <= 'z') {
                    d = *p - 'a' + 10;
                }
                else if (*p >= 'A' && *p <= 'Z') {
                    d = *p - 'A' + 10;
                }
                else {
                    break;
                }

                if (d >= r) {
                    break;
                }

                if (n * r + d < n) {
                    error("integer overflow");
                    break;
                }

                n = n * r + d;
                p++;
            }

            // if n needs more than 64 bits
            if (n.sizeof > 8 && n > 0xffffffffffffffffL) {
                error("integer overflow");
            }
        }

        // Parse trailing 'u', 'U', 'l' or 'L' in any combination
        while (true) {
            ubyte f;
            switch (*p) {
            case 'U':
            case 'u':
                f = FLAGS.FLAGS_unsigned;
                goto L1;

            case 'L':
            case 'l':
                f = FLAGS.FLAGS_long;
L1:
                p++;
                if (flags & f) {
                    error("unrecognized token");
                }
                flags = cast(FLAGS)(flags | f);
                continue;

            default:
                break;
            }
            break;
        }

        switch (flags) {
        case 0:
            /* Octal or Hexadecimal constant.
             * First that fits: int, uint, long, ulong
             */
            if (n & 0x8000000000000000L) {
                result = TOK.TOKuns64v;
            }
            else if (n & 0xffffffff00000000L) {
                result = TOK.TOKint64v;
            }
            else if (n & 0x80000000) {
                result = TOK.TOKuns32v;
            }
            else {
                result = TOK.TOKint32v;
            }
            break;

        case FLAGS.FLAGS_decimal:
            /* First that fits: int, long, long long
             */
            if (n & 0x8000000000000000L) {
                error("signed integer overflow");
                result = TOK.TOKuns64v;
            }
            else if (n & 0xffffffff80000000L) {
                result = TOK.TOKint64v;
            }
            else {
                result = TOK.TOKint32v;
            }
            break;

        case FLAGS.FLAGS_unsigned:
        case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned:
            /* First that fits: uint, ulong
             */
            if (n & 0xffffffff00000000L) {
                result = TOK.TOKuns64v;
            }
            else {
                result = TOK.TOKuns32v;
            }
            break;

        case FLAGS.FLAGS_decimal | FLAGS.FLAGS_long:
            if (n & 0x8000000000000000L) {
                error("signed integer overflow");
                result = TOK.TOKuns64v;
            }
            else {
                result = TOK.TOKint64v;
            }
            break;

        case FLAGS.FLAGS_long:
            if (n & 0x8000000000000000L) {
                result = TOK.TOKuns64v;
            }
            else {
                result = TOK.TOKint64v;
            }
            break;

        case FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long:
        case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long:
            result = TOK.TOKuns64v;
            break;

        default:
            debug writefln("%x", flags);
            assert(0);
        }
        t.uns64value = n;
        return(result);
    }

    /**************************************
     * Read in characters, converting them to real.
     * Bugs:
     *	Exponent overflow not detected.
     *	Too much requested precision is not detected.
     */

    TOK inreal(Token *t) {
        int  dblstate;
        uint c;
        char hex;                       // is this a hexadecimal-floating-constant?
        TOK  result;

        //printf("Lexer.inreal()\n");
        stringbuffer.offset = 0;
        dblstate            = 0;
        hex                 = 0;
Lnext:
        while (1) {
            // Get next char from input
            c = *p++;
            //printf("dblstate = %d, c = '%s'\n", dblstate, c);
            while (1) {
                switch (dblstate) {
                case 0:                                 // opening state
                    if (c == '0') {
                        dblstate = 9;
                    }
                    else if (c == '.') {
                        dblstate = 3;
                    }
                    else {
                        dblstate = 1;
                    }
                    break;

                case 9:
                    dblstate = 1;
                    if (c == 'X' || c == 'x') {
                        hex++;
                        break;
                    }

                case 1:                                 // digits to left of .
                case 3:                                 // digits to right of .
                case 7:                                 // continuing exponent digits
                    if (!isdigit(c) && !(hex && isxdigit(c))) {
                        if (c == '_') {
                            goto Lnext;                         // ignore embedded '_'
                        }
                        dblstate++;
                        continue;
                    }
                    break;

                case 2:                                 // no more digits to left of .
                    if (c == '.') {
                        dblstate++;
                        break;
                    }

                case 4:                                 // no more digits to right of .
                    if ((c == 'E' || c == 'e') || hex && (c == 'P' || c == 'p')) {
                        dblstate = 5;
                        hex      = 0;                           // exponent is always decimal
                        break;
                    }
                    if (hex) {
                        error("binary-exponent-part required");
                    }
                    goto done;

                case 5:                                 // looking immediately to right of E
                    dblstate++;
                    if (c == '-' || c == '+') {
                        break;
                    }

                case 6:                                 // 1st exponent digit expected
                    if (!isdigit(c)) {
                        error("exponent expected");
                    }
                    dblstate++;
                    break;

                case 8:                                 // past end of exponent digits
                    goto done;
                }
                break;
            }
            stringbuffer.write(c);
        }
done:
        p--;

        stringbuffer.write(cast(byte)0);

//	#if _WIN32 && __DMC__
        char *save = __locale_decpoint;
        __locale_decpoint = ".";
//	#endif
        t.float80value = strtold(cast(char *)stringbuffer.data.ptr, null);
        errno          = 0;
        switch (*p) {
        case 'F':
        case 'f':
            strtof(cast(char *)stringbuffer.data.ptr, null);
            result = TOK.TOKfloat32v;
            p++;
            break;

        default:
            strtod(cast(char *)stringbuffer.data.ptr, null);
            result = TOK.TOKfloat64v;
            break;

        case 'L':
        case 'l':
            result = TOK.TOKfloat80v;
            p++;
            break;
        }
        if (*p == 'i' || *p == 'I') {
            p++;
            switch (result) {
            case TOK.TOKfloat32v:
                result = TOK.TOKimaginary32v;
                break;

            case TOK.TOKfloat64v:
                result = TOK.TOKimaginary64v;
                break;

            case TOK.TOKfloat80v:
                result = TOK.TOKimaginary80v;
                break;
            }
        }
//	#if _WIN32 && __DMC__
        __locale_decpoint = save;
//	#endif
        if (errno == ERANGE) {
            error("number is not representable");
        }
        return(result);
    }




    /*********************************************
     * Do pragma.
     * Currently, the only pragma supported is:
     *	#line linnum [filespec]
     */

    void Pragma() {
        Token  tok;
        int    linnum;
        char[] filespec;
        Loc    loc = this.loc;

        scan(&tok);

        if (tok.value != TOK.TOKidentifier || tok.identifier != Id.line) {
            goto Lerr;
        }

        scan(&tok);
        if (tok.value == TOK.TOKint32v || tok.value == TOK.TOKint64v) {
            linnum = tok.uns64value - 1;
        }
        else {
            goto Lerr;
        }

        while (1) {
            switch (*p) {
            case 0:
            case 0x1a:
            case '\n':
Lnewline:
                this.loc.linnum = linnum;
                if (filespec.length) {
                    this.loc.filename = filespec;
                }
                return;

            case '\r':
                p++;
                if (*p != '\n') {
                    p--;
                    goto Lnewline;
                }
                continue;

            case ' ':
            case '\t':
            case '\v':
            case '\f':
                p++;
                continue;                                               // skip white space

            case '_':
                if (mod && memcmp(p, cast(char*)"__FILE__", 8) == 0) {
                    p += 8;
//!					    filespec = mem.strdup(loc.filename ? loc.filename : mod.identifier.toChars());
                }
                continue;

            case '"':
                if (filespec) {
                    goto Lerr;
                }
                stringbuffer.offset = 0;
                p++;
                while (1) {
                    uint c;
                    c = *p;
                    switch (c) {
                    case '\n':
                    case '\r':
                    case 0:
                    case 0x1a:
                        goto Lerr;

                    case '"':
                        stringbuffer.write(cast(byte)0);
                        //						    filespec = mem.strdup((char *)stringbuffer.data);
                        filespec = stringbuffer.toString.dup;
                        p++;
                        break;

                    default:
                        if (c & 0x80) {
                            uint u = decodeUTF();
                            if (u == PS || u == LS) {
                                goto Lerr;
                            }
                        }
                        stringbuffer.write(c);
                        p++;
                        continue;
                    }
                    break;
                }
                continue;

            default:
                if (*p & 0x80) {
                    uint u = decodeUTF();
                    if (u == PS || u == LS) {
                        goto Lnewline;
                    }
                }
                goto Lerr;
            }
        }

Lerr:
        errorLoc(loc, "#line integer [\"filespec\"]\\n expected");
    }



    /***************************************************
     * Parse doc comment embedded between t.ptr and p.
     * Remove trailing blanks and tabs from lines.
     * Replace all newlines with \n.
     * Remove leading comment character from each line.
     * Decide if it's a lineComment or a blockComment.
     * Append to previous one for this token.
     */

    void getDocComment(Token *t, uint lineComment) {
        auto OutBuffer buf       = new OutBuffer;
        ubyte          ct        = t.ptr[2];
        ubyte          *q        = t.ptr + 3; // start of comment text
        int            linestart = 0;

        ubyte          *qend = p;

        if (ct == '*' || ct == '+') {
            qend -= 2;
        }

        // Scan over initial row of ****'s or ++++'s or ////'s
        for (; q < qend; q++) {
            if (*q != ct) {
                break;
            }
        }

        // Remove trailing row of ****'s or ++++'s
        if (ct != '/') {
            for (; q < qend; qend--) {
                if (qend[-1] != ct) {
                    break;
                }
            }
        }

        for (; q < qend; q++) {
            ubyte c = *q;

            switch (c) {
            case '*':
            case '+':
                if (linestart && c == ct) {
                    linestart = 0;
                    // Trim preceding whitespace up to preceding \n
                    while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) {
                        buf.offset--;
                    }
                    continue;
                }
                break;

            case ' ':
            case '\t':
                break;

            case '\r':
                if (q[1] == '\n') {
                    continue;                                   // skip the \r
                }
                goto Lnewline;

            default:
                if (c == 226) {
                    // If LS or PS
                    if (q[1] == 128 &&
                        (q[2] == 168 || q[2] == 169)) {
                        q += 2;
                        goto Lnewline;
                    }
                }
                linestart = 0;
                break;

Lnewline:
                c = '\n';                               // replace all newlines with \n

            case '\n':
                linestart = 1;

                // Trim trailing whitespace
                while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) {
                    buf.offset--;
                }

                break;
            }
            buf.write(c);
        }

        // Always end with a newline
        if (!buf.offset || buf.data[buf.offset - 1] != '\n') {
            buf.writenl();
        }

        //buf.write(cast(char)0);

        // It's a line comment if the start of the doc comment comes
        // after other non-whitespace on the same line.
//	    ubyte** dc = (lineComment && anyToken)
//				 ? &t.lineComment
//				 : &t.blockComment;

        char[] dc = (lineComment && anyToken) ? t.lineComment : t.blockComment;

        // Combine with previous doc comment, if any
        if (dc.length) {
            dc = combineComments(dc, buf.toString().dup);
        }
        else {
            dc = buf.toString().dup;
        }

//		writefln( dc );

        if (lineComment && anyToken) {
            t.lineComment = dc;
        }
        else {
            t.blockComment = dc;
        }
    }
}

// character maps
static ubyte[256] cmtable;

const int CMoctal  = 0x1;
const int CMhex    = 0x2;
const int CMidchar = 0x4;

ubyte isoctal(ubyte c) {
    return(cmtable[c] & CMoctal);
}
ubyte ishex(ubyte c) {
    return(cmtable[c] & CMhex);
}
ubyte isidchar(ubyte c) {
    return(cmtable[c] & CMidchar);
}

static void cmtable_init() {
    for (uint c = 0; c < cmtable.length; c++) {
        if ('0' <= c && c <= '7') {
            cmtable[c] |= CMoctal;
        }
        if (isdigit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F')) {
            cmtable[c] |= CMhex;
        }
        if (isalnum(c) || c == '_') {
            cmtable[c] |= CMidchar;
        }
    }
}


/+
 * struct StringValue
 * {
 *  union
 *  {
 *              int intvalue;
 *              void *ptrvalue;
 *              dchar *string;
 *  }
 *
 *  char[] lstring;
 * }
 * #define CASE_BASIC_TYPES
 *      case TOKwchar: case TOKdchar:
 *      case TOKbit: case TOKbool: case TOKchar:
 *      case TOKint8: case TOKuns8:
 *      case TOKint16: case TOKuns16:
 *      case TOKint32: case TOKuns32:
 *      case TOKint64: case TOKuns64:
 *      case TOKfloat32: case TOKfloat64: case TOKfloat80:
 *      case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:
 *      case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:
 *      case TOKvoid:
 *
 * #define CASE_BASIC_TYPES_X(t)					\
 *      case TOKvoid:	 t = Type::tvoid;  goto LabelX;		\
 *      case TOKint8:	 t = Type::tint8;  goto LabelX;		\
 *      case TOKuns8:	 t = Type::tuns8;  goto LabelX;		\
 *      case TOKint16:	 t = Type::tint16; goto LabelX;		\
 *      case TOKuns16:	 t = Type::tuns16; goto LabelX;		\
 *      case TOKint32:	 t = Type::tint32; goto LabelX;		\
 *      case TOKuns32:	 t = Type::tuns32; goto LabelX;		\
 *      case TOKint64:	 t = Type::tint64; goto LabelX;		\
 *      case TOKuns64:	 t = Type::tuns64; goto LabelX;		\
 *      case TOKfloat32: t = Type::tfloat32; goto LabelX;	\
 *      case TOKfloat64: t = Type::tfloat64; goto LabelX;	\
 *      case TOKfloat80: t = Type::tfloat80; goto LabelX;	\
 *      case TOKimaginary32: t = Type::timaginary32; goto LabelX;	\
 *      case TOKimaginary64: t = Type::timaginary64; goto LabelX;	\
 *      case TOKimaginary80: t = Type::timaginary80; goto LabelX;	\
 *      case TOKcomplex32: t = Type::tcomplex32; goto LabelX;	\
 *      case TOKcomplex64: t = Type::tcomplex64; goto LabelX;	\
 *      case TOKcomplex80: t = Type::tcomplex80; goto LabelX;	\
 *      case TOKbit:	 t = Type::tbit;     goto LabelX;	\
 *      case TOKchar:	 t = Type::tchar;    goto LabelX;	\
 *      case TOKwchar:	 t = Type::twchar; goto LabelX;	\
 *      case TOKdchar:	 t = Type::tdchar; goto LabelX;	\
 *      LabelX
 +/