/* * TokenMarker.java - Generic token marker * Copyright (C) 1998, 1999 Slava Pestov * * You may use and modify this package for any purpose. Redistribution is * permitted, in both source and binary form, provided that this notice * remains intact in all source distributions of this package. */package org.syntax.jedit.tokenmarker;import javax.swing.text.Segment;import java.util.*;/** * A token marker that splits lines of text into tokens. Each token carries * a length field and an indentification tag that can be mapped to a color * for painting that token.<p> * * For performance reasons, the linked list of tokens is reused after each * line is tokenized. Therefore, the return value of <code>markTokens</code> * should only be used for immediate painting. Notably, it cannot be * cached. * * @author Slava Pestov * @version $Id: TokenMarker.java,v 1.32 1999/12/13 03:40:30 sp Exp $ * * @see org.syntax.jedit.Token */public abstract class TokenMarker{	/**	 * A wrapper for the lower-level <code>markTokensImpl</code> method	 * that is called to split a line up into tokens.	 * @param line The line	 * @param lineIndex The line number	 */	public Token markTokens(Segment line, int lineIndex)	{		if(lineIndex >= length)		{			throw new IllegalArgumentException("Tokenizing invalid line: "				+ lineIndex);		}		lastToken = null;		LineInfo info = lineInfo[lineIndex];		LineInfo prev;		if(lineIndex == 0)			prev = null;		else			prev = lineInfo[lineIndex - 1];		byte oldToken = info.token;		byte token = markTokensImpl(prev == null ?			Token.NULL : prev.token,line,lineIndex);		info.token = token;		/*		 * This is a foul hack. It stops nextLineRequested		 * from being cleared if the same line is marked twice.		 *		 * Why is this necessary? It's all JEditTextArea's fault.		 * When something is inserted into the text, firing a		 * document event, the insertUpdate() method shifts the		 * caret (if necessary) by the amount inserted.		 *		 * All caret movement is handled by the select() method,		 * which eventually pipes the new position to scrollTo()		 * and calls repaint().		 *		 * Note that at this point in time, the new line hasn't		 * yet been painted; the caret is moved first.		 *		 * scrollTo() calls offsetToX(), which tokenizes the line		 * unless it is being called on the last line painted		 * (in which case it uses the text area's painter cached		 * token list). What scrollTo() does next is irrelevant.		 *		 * After scrollTo() has done it's job, repaint() is		 * called, and eventually we end up in paintLine(), whose		 * job is to paint the changed line. It, too, calls		 * markTokens().		 *		 * The problem was that if the line started a multiline		 * token, the first markTokens() (done in offsetToX())		 * would set nextLineRequested (because the line end		 * token had changed) but the second would clear it		 * (because the line was the same that time) and therefore		 * paintLine() would never know that it needed to repaint		 * subsequent lines.		 *		 * This bug took me ages to track down, that's why I wrote		 * all the relevant info down so that others wouldn't		 * duplicate it.		 */		 if(!(lastLine == lineIndex && nextLineRequested))			nextLineRequested = (oldToken != token);		lastLine = lineIndex;		addToken(0,Token.END);		return firstToken;	}	/**	 * An abstract method that splits a line up into tokens. It	 * should parse the line, and call <code>addToken()</code> to	 * add syntax tokens to the token list. Then, it should return	 * the initial token type for the next line.<p>	 *	 * For example if the current line contains the start of a 	 * multiline comment that doesn't end on that line, this method	 * should return the comment token type so that it continues on	 * the next line.	 *	 * @param token The initial token type for this line	 * @param line The line to be tokenized	 * @param lineIndex The index of the line in the document,	 * starting at 0	 * @return The initial token type for the next line	 */	protected abstract byte markTokensImpl(byte token, Segment line,		int lineIndex);	/**	 * Returns if the token marker supports tokens that span multiple	 * lines. If this is true, the object using this token marker is	 * required to pass all lines in the document to the	 * <code>markTokens()</code> method (in turn).<p>	 *	 * The default implementation returns true; it should be overridden	 * to return false on simpler token markers for increased speed.	 */	public boolean supportsMultilineTokens()	{		return true;	}	/**	 * Informs the token marker that lines have been inserted into	 * the document. This inserts a gap in the <code>lineInfo</code>	 * array.	 * @param index The first line number	 * @param lines The number of lines 	 */	public void insertLines(int index, int lines)	{		if(lines <= 0)			return;		length += lines;		ensureCapacity(length);		int len = index + lines;		System.arraycopy(lineInfo,index,lineInfo,len,			lineInfo.length - len);		for(int i = index + lines - 1; i >= index; i--)		{			lineInfo[i] = new LineInfo();		}	}		/**	 * Informs the token marker that line have been deleted from	 * the document. This removes the lines in question from the	 * <code>lineInfo</code> array.	 * @param index The first line number	 * @param lines The number of lines	 */	public void deleteLines(int index, int lines)	{		if (lines <= 0)			return;		int len = index + lines;		length -= lines;		System.arraycopy(lineInfo,len,lineInfo,			index,lineInfo.length - len);	}	/**	 * Returns the number of lines in this token marker.	 */	public int getLineCount()	{		return length;	}	/**	 * Returns true if the next line should be repainted. This	 * will return true after a line has been tokenized that starts	 * a multiline token that continues onto the next line.	 */	public boolean isNextLineRequested()	{		return nextLineRequested;	}	// protected members	/**	 * The first token in the list. This should be used as the return	 * value from <code>markTokens()</code>.	 */	protected Token firstToken;	/**	 * The last token in the list. New tokens are added here.	 * This should be set to null before a new line is to be tokenized.	 */	protected Token lastToken;	/**	 * An array for storing information about lines. It is enlarged and	 * shrunk automatically by the <code>insertLines()</code> and	 * <code>deleteLines()</code> methods.	 */	protected LineInfo[] lineInfo;	/**	 * The number of lines in the model being tokenized. This can be	 * less than the length of the <code>lineInfo</code> array.	 */	protected int length;	/**	 * The last tokenized line.	 */	protected int lastLine;	/**	 * True if the next line should be painted.	 */	protected boolean nextLineRequested;	/**	 * Creates a new <code>TokenMarker</code>. This DOES NOT create	 * a lineInfo array; an initial call to <code>insertLines()</code>	 * does that.	 */	protected TokenMarker()	{		lastLine = -1;	}	/**	 * Ensures that the <code>lineInfo</code> array can contain the	 * specified index. This enlarges it if necessary. No action is	 * taken if the array is large enough already.<p>	 *	 * It should be unnecessary to call this under normal	 * circumstances; <code>insertLine()</code> should take care of	 * enlarging the line info array automatically.	 *	 * @param index The array index	 */	protected void ensureCapacity(int index)	{		if(lineInfo == null)			lineInfo = new LineInfo[index + 1];		else if(lineInfo.length <= index)		{			LineInfo[] lineInfoN = new LineInfo[(index + 1) * 2];			System.arraycopy(lineInfo,0,lineInfoN,0,					 lineInfo.length);			lineInfo = lineInfoN;		}	}	/**	 * Adds a token to the token list.	 * @param length The length of the token	 * @param id The id of the token	 */	protected void addToken(int length, byte id)	{		if(id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST)			throw new InternalError("Invalid id: " + id);		if(length == 0 && id != Token.END)			return;		if(firstToken == null)		{			firstToken = new Token(length,id);			lastToken = firstToken;		}		else if(lastToken == null)		{			lastToken = firstToken;			firstToken.length = length;			firstToken.id = id;		}		else if(lastToken.next == null)		{			lastToken.next = new Token(length,id);			lastToken = lastToken.next;		}		else		{			lastToken = lastToken.next;			lastToken.length = length;			lastToken.id = id;		}	}	/**	 * Inner class for storing information about tokenized lines.	 */	public class LineInfo	{		/**		 * Creates a new LineInfo object with token = Token.NULL		 * and obj = null.		 */		public LineInfo()		{		}		/**		 * Creates a new LineInfo object with the specified		 * parameters.		 */		public LineInfo(byte token, Object obj)		{			this.token = token;			this.obj = obj;		}		/**		 * The id of the last token of the line.		 */		public byte token;		/**		 * This is for use by the token marker implementations		 * themselves. It can be used to store anything that		 * is an object and that needs to exist on a per-line		 * basis.		 */		public Object obj;	}}