// HTMLParser Library $Name: v1_6_20051112 $ - A java-based parser for HTML// http://sourceforge.org/projects/htmlparser// Copyright (C) 2004 Derrick Oswald//// Revision Control Information//// $Source: /cvsroot/htmlparser/htmlparser/src/org/htmlparser/tests/lexerTests/LexerTests.java,v $// $Author: derrickoswald $// $Date: 2005/11/12 16:44:54 $// $Revision: 1.27 $//// This library is free software; you can redistribute it and/or// modify it under the terms of the GNU Lesser General Public// License as published by the Free Software Foundation; either// version 2.1 of the License, or (at your option) any later version.//// This library is distributed in the hope that it will be useful,// but WITHOUT ANY WARRANTY; without even the implied warranty of// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU// Lesser General Public License for more details.//// You should have received a copy of the GNU Lesser General Public// License along with this library; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA//package org.htmlparser.tests.lexerTests;import java.io.IOException;import java.net.URL;import java.util.HashSet;import org.htmlparser.Node;import org.htmlparser.Parser;import org.htmlparser.Remark;import org.htmlparser.Tag;import org.htmlparser.Text;import org.htmlparser.lexer.Lexer;import org.htmlparser.tags.ScriptTag;import org.htmlparser.tags.StyleTag;import org.htmlparser.tests.ParserTestCase;import org.htmlparser.util.EncodingChangeException;import org.htmlparser.util.NodeIterator;import org.htmlparser.util.NodeList;import org.htmlparser.util.ParserException;public class LexerTests extends ParserTestCase{    static    {        System.setProperty ("org.htmlparser.tests.lexerTests.LexerTests", "LexerTests");    }    /**     * Test the Lexer class.     */    public LexerTests (String name)    {        super (name);    }    /**     * Test operation without tags.     */    public void testPureText () throws ParserException    {        String reference;        Lexer lexer;        Text node;        reference = "Hello world";        lexer = new Lexer (reference);        node = (Text)lexer.nextNode ();        assertEquals ("Text contents wrong", reference, node.getText ());    }    /**     * Test operation with Unix line endings.     */    public void testUnixEOL () throws ParserException    {        String reference;        Lexer lexer;        Text node;        reference = "Hello\nworld";        lexer = new Lexer (reference);        node = (Text)lexer.nextNode ();        assertEquals ("Text contents wrong", reference, node.getText ());    }    /**     * Test operation with Dos line endings.     */    public void testDosEOL () throws ParserException    {        String reference;        Lexer lexer;        Text node;        reference = "Hello\r\nworld";        lexer = new Lexer (reference);        node = (Text)lexer.nextNode ();        assertEquals ("Text contents wrong", reference, node.getText ());        reference = "Hello\rworld";        lexer = new Lexer (reference);        node = (Text)lexer.nextNode ();        assertEquals ("Text contents wrong", reference, node.getText ());    }    /**     * Test operation with line endings near the end of input.     */    public void testEOF_EOL () throws ParserException    {        String reference;        Lexer lexer;        Text node;        reference = "Hello world\n";        lexer = new Lexer (reference);        node = (Text)lexer.nextNode ();        assertEquals ("Text contents wrong", reference, node.getText ());        reference = "Hello world\r";        lexer = new Lexer (reference);        node = (Text)lexer.nextNode ();        assertEquals ("Text contents wrong", reference, node.getText ());        reference = "Hello world\r\n";        lexer = new Lexer (reference);        node = (Text)lexer.nextNode ();        assertEquals ("Text contents wrong", reference, node.getText ());    }    /**     * Test that tags stop string nodes.     */    public void testTagStops () throws ParserException    {        String[] references =        {            "Hello world",            "Hello world\n",            "Hello world\r\n",            "Hello world\r",        };        String[] suffixes =        {            "<head>",            "</head>",            "<%=head%>",            "<!--head-->",        };        Lexer lexer;        Text node;        for (int i = 0; i < references.length; i++)        {            for (int j = 0; j < suffixes.length; j++)            {                lexer = new Lexer (references[i] + suffixes[j]);                node = (Text)lexer.nextNode ();                assertEquals ("Text contents wrong", references[i], node.getText ());            }        }    }    /**     * Test operation with only tags.     */    public void testPureTag () throws ParserException    {        String reference;        String suffix;        Lexer lexer;        Node node;        reference = "<head>";        lexer = new Lexer (reference);        node = lexer.nextNode ();        assertEquals ("Tag contents wrong", reference, node.toHtml ());        reference = "<head>";        suffix = "<body>";        lexer = new Lexer (reference + suffix);        node = lexer.nextNode ();        assertEquals ("Tag contents wrong", reference, node.toHtml ());        node = lexer.nextNode ();        assertEquals ("Tag contents wrong", suffix, node.toHtml ());    }    /**     * Test operation with attributed tags.     */    public void testAttributedTag () throws ParserException    {        String reference;        Lexer lexer;        Node node;        reference = "<head lang='en_US' dir=ltr\nprofile=\"http://htmlparser.sourceforge.org/dictionary.html\">";        lexer = new Lexer (reference);        node = lexer.nextNode ();        assertEquals ("Tag contents wrong", reference, node.toHtml ());    }    /**     * Test operation with comments.     */    public void testRemark () throws ParserException    {        String reference;        Lexer lexer;        Remark node;        String suffix;        reference = "<!-- This is a comment -->";        lexer = new Lexer (reference);        node = (Remark)lexer.nextNode ();        assertEquals ("Tag contents wrong", reference, node.toHtml ());        reference = "<!-- This is a comment --  >";        lexer = new Lexer (reference);        node = (Remark)lexer.nextNode ();        assertEquals ("Tag contents wrong", reference, node.toHtml ());        reference = "<!-- This is a\nmultiline comment -->";        lexer = new Lexer (reference);        node = (Remark)lexer.nextNode ();        assertEquals ("Tag contents wrong", reference, node.toHtml ());        suffix = "<head>";        reference = "<!-- This is a comment -->";        lexer = new Lexer (reference + suffix);        node = (Remark)lexer.nextNode ();        assertEquals ("Tag contents wrong", reference, node.toHtml ());        reference = "<!-- This is a comment --  >";        lexer = new Lexer (reference + suffix);        node = (Remark)lexer.nextNode ();        assertEquals ("Tag contents wrong", reference, node.toHtml ());        reference = "<!-- This is a\nmultiline comment -->";        lexer = new Lexer (reference + suffix);        node = (Remark)lexer.nextNode ();        assertEquals ("Tag contents wrong", reference, node.toHtml ());    }//    /**//     * Try a real page.//     *///    public void testReal () throws ParserException, IOException//    {//        Lexer lexer;//        Node node;////        URL url = new URL ("http://sourceforge.net/projects/htmlparser");//        lexer = new Lexer (url.openConnection ());//        while (null != (node = lexer.nextNode ()))//            System.out.println (node.toString ());//    }    /**     * Test the fidelity of the toHtml() method.     */    public void testFidelity () throws ParserException, IOException    {        Lexer lexer;        Node node;        int position;        StringBuffer buffer;        String string;        char[] ref;        char[] test;        URL url = new URL ("http://sourceforge.net");        lexer = new Lexer (url.openConnection ());        position = 0;        buffer = new StringBuffer (80000);        while (null != (node = lexer.nextNode ()))        {            string = node.toHtml ();            if (position != node.getStartPosition ())                fail ("non-contiguous" + string);            buffer.append (string);            position = node.getEndPosition ();            if (buffer.length () != position)                fail ("text length differed after encountering node " + string);        }        ref = lexer.getPage ().getText ().toCharArray ();        test = new char[buffer.length ()];        buffer.getChars (0, buffer.length (), test, 0);        assertEquals ("different amounts of text", ref.length, test.length);        for (int i = 0; i < ref.length; i++)            if (ref[i] != test[i])                fail ("character differs at position " + i + ", expected <" + ref[i] + "> but was <" + test[i] + ">");    }//    /**//     * Test the relative speed reading from a string parsing tags too.//     *///    public void testSpeedStringWithoutTags () throws ParserException, IOException