// released into the public domain
// by Christopher Diggins 2004
// http://www.cdiggins.com

#include <iostream>

#include "yard.hpp"
#include "utils.hpp"

using namespace std;
using namespace yard;

void RunTokenizerTest() {
    std::cout << "testing tokenizer";

    char * pBuf = AllocFromFile("test.txt");
	if(NULL==pBuf)
		return;
    const char * pEnd = & (pBuf[strlen(pBuf)]);

    Tokenizer < MatchWord > tknzr;
    {
        TimeIt t;
        tknzr.Tokenize(pBuf, pEnd);
    }

    OutputTokens(tknzr.begin(), tknzr.end());
    free(pBuf);
}

int main() {
    RunTokenizerTest();
    return 0;
}