lAddByte(pTok, (unsigned char)(yylval.sc_int ? 1 : 0));
    default:
        break;
    }
} // RecordToken

/*
 * RewindTokenStream() - Reset a token stream in preperation for reading.
 *
 */

void RewindTokenStream(TokenStream *pTok)
{
    if (pTok->head) {
        pTok->current = pTok->head;
        pTok->current->current = 0;
    }
} // RewindTokenStream

/*
 * ReadToken() - Read the next token from a stream.
 *
 */

int ReadToken(TokenStream *pTok)
{
    char symbol_name[MAX_SYMBOL_NAME_LEN + 1];
    char string_val[MAX_STRING_LEN + 1];
    int ltoken, len;
    char ch;

    ltoken = lReadByte(pTok);
    if (ltoken >= 0) {
        if (ltoken > 127)
            ltoken += 128;
        //ltoken = lExpandToken(ltoken);
        switch (ltoken) {
        case IDENT_SY:
        case TYPEIDENT_SY:
            len = 0;
            ch = lReadByte(pTok);
            while ((ch >= 'a' && ch <= 'z') ||
                     (ch >= 'A' && ch <= 'Z') ||
                     (ch >= '0' && ch <= '9') ||
                     ch == '_')
            {
                if (len < MAX_SYMBOL_NAME_LEN) {
                    symbol_name[len] = ch;
                    len++;
                    ch = lReadByte(pTok);
                }
            }
            symbol_name[len] = '\0';
            assert(ch == '\0');
            yylval.sc_ident = LookUpAddString(atable, symbol_name);
            return IDENT_SY;
            break;
        case STRCONST_SY:
            len = 0;
            while ((ch = lReadByte(pTok)) != 0)
                if (len < MAX_STRING_LEN)
                    string_val[len++] = ch;
            string_val[len] = 0;
            yylval.sc_ident = LookUpAddString(atable, string_val);
            break;
        case CFLOATCONST_SY:
        case FLOATCONST_SY:
        case FLOATHCONST_SY:
        case FLOATXCONST_SY:
            lRead4Bytes(pTok, (unsigned char *) &yylval.sc_fval);
            break;
        case INTCONST_SY:
            lRead4Bytes(pTok, (unsigned char *) &yylval.sc_int);
            break;
        case '(':
            yylval.sc_int = lReadByte(pTok);
            break;
        }
        return ltoken;
    }
    return EOF_SY;
} // ReadToken

typedef struct TokenInputSrc {
    InputSrc            base;
    TokenStream         *tokens;
    int                (*final)(CgStruct *);
} TokenInputSrc;

static int scan_token(TokenInputSrc *in)
{
    int token = ReadToken(in->tokens);
    int (*final)(CgStruct *);
    Cg->tokenLoc->file = Cg->currentInput->name;
    Cg->tokenLoc->line = Cg->currentInput->line;
    if (token == '\n') {
        in->base.line++;
        //printf("    end of line %d\n", tokenloc->line);
        return token;
    }
    if (token > 0) return token;
    Cg->currentInput = in->base.prev;
    final = in->final;
    free(in);
    if (final && !final(Cg)) return -1;
    return Cg->currentInput->scan(Cg->currentInput);
}

int ReadFromTokenStream(TokenStream *ts, int name, int (*final)(CgStruct *))
{
    TokenInputSrc *in = malloc(sizeof(TokenInputSrc));
    memset(in, 0, sizeof(TokenInputSrc));
    in->base.name = name;
    in->base.prev = Cg->currentInput;
    in->base.scan = (int (*)(InputSrc *))scan_token;
    in->base.line = 1;
    in->tokens = ts;
    in->final = final;
    RewindTokenStream(ts);
    Cg->currentInput = &in->base;
    return 1;
}

typedef struct UngotToken {
    InputSrc    base;
    int         token;
    YYSTYPE     lval;
} UngotToken;

static int reget_token(UngotToken *t)
{
    int token = t->token;
    yylval = t->lval;
    Cg->currentInput = t->base.prev;
    free(t);
    return token;
}

void UngetToken(int token) {
    UngotToken *t = malloc(sizeof(UngotToken));
    memset(t, 0, sizeof(UngotToken));
    t->token = token;
    t->lval = yylval;
    t->base.scan = (void *)reget_token;
    t->base.prev = Cg->currentInput;
    t->base.name = Cg->currentInput->name;
    t->base.line = Cg->currentInput->line;
    Cg->currentInput = &t->base;
}

///////////////////////////////////// Tokenize Input File: ////////////////////////////////////

#if defined(CGC_ENABLE_TOOLS)

void TokenizeInput(void)
{
    int ltoken, index, count;

    TokenStream *RecordedTokens = NewTokenStream(Cg->options.sourceFileName);
    while ((ltoken = Cg->currentInput->scan(Cg->currentInput)) > 0 &&
           ltoken != ERROR_SY
    ) {
        RecordToken(RecordedTokens, ltoken);
    }

    // Debug print stuff to screen:

#if 0
    RewindTokenStream(RecordedTokens);
    ltoken = ReadToken(RecordedTokens);
    while (ltoken != EOF_SY && ltoken != ERROR_SY) {
        if (ltoken >= 127)
            printf("token = %s", GetAtomString(atable, ltoken));
        else if (ltoken >= 32) 
            printf("token = <%c>", ltoken);
        else if (ltoken == '\n')
            printf("token = <\\n>");
        else if (ltoken > 0)
            printf("token = <\\0%o>", ltoken);
        else
            printf("token = <EOF>");
        switch (ltoken) {
        case IDENT_SY:
        case TYPEIDENT_SY:
        case STRCONST_SY:
            printf(" = \"%s\"", GetAtomString(atable, yylval.sc_ident));
            break;
        case CFLOATCONST_SY:
        case FLOATCONST_SY:
        case FLOATHCONST_SY:
        case FLOATXCONST_SY:
            printf(" = %g9.6", yylval.sc_fval);
            break;
        case INTCONST_SY:
            printf(" = %d", yylval.sc_int);
            break;
        }
        printf("\n");
        ltoken = ReadToken(RecordedTokens);
    }
#endif

    // Dump file to screen as a C initialization statement:

    printf("// automatically generated from %s -- do no edit\n\n",
           Cg->options.sourceFileName);
    printf("#include <stdio.h>\n");
    printf("#include \"slglobals.h\"\n\n");
    printf("unsigned char %s_tokendata[] = {\n", RecordedTokens->name);
    RewindTokenStream(RecordedTokens);
    ltoken = lReadByte(RecordedTokens);
    index = count = 0;
    while (ltoken != EOF_SY && ltoken != ERROR_SY) {
        if (index == 0)
            printf("    ");
        printf("0x%02x,", ltoken);
        count++;
        index++;
        if (index >= 16) {
            printf("\n");
            index = 0;
        }
        ltoken = lReadByte(RecordedTokens);
    }
    if (index > 0)
        printf("\n");
    printf("};");
    printf("\n");
    printf("TokenBlock %s_blockdata = {\n", RecordedTokens->name);
    printf("    NULL, // next\n");
    printf("    0,    // current\n");
    printf("    %d, // count\n", count);
    printf("    %d, // max\n", count);
    printf("    %s_tokendata // data\n", RecordedTokens->name);
    printf("};\n");

    printf("TokenStream %s_stream = { \n", RecordedTokens->name);
    printf("    NULL, // next\n"
           "    \"%s\", // name\n", Cg->options.sourceFileName);
    printf("    &%s_blockdata, // head\n", RecordedTokens->name);
    printf("    &%s_blockdata // current\n", RecordedTokens->name);
    printf("};\n");

} // TokenizeInput

void DumpTokenStream(FILE *fp, TokenStream *s) {
    int token;

    if (fp == 0) fp = stdout;
    RewindTokenStream(s);
    while ((token = ReadToken(s)) > 0) {
        switch (token) {
        case IDENT_SY:
        case TYPEIDENT_SY:
            printf("%s ", GetAtomString(atable, yylval.sc_ident));
            break;
        case STRCONST_SY:
            printf("\"%s\"", GetAtomString(atable, yylval.sc_ident));
            break;
        case CFLOATCONST_SY:
        case FLOATCONST_SY:
        case FLOATHCONST_SY:
        case FLOATXCONST_SY:
            printf("%g9.6 ", yylval.sc_fval);
            break;
        case INTCONST_SY:
            printf("%d ", yylval.sc_int);
            break;
        default:
            if (token >= 127)
                printf("%s ", GetAtomString(atable, token));
            else
                printf("%c", token);
            break;
        }
    }
}
#endif // defined(CGC_ENABLE_TOOLS)

///////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////// End of tokens.c ///////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////