public final int yychar(){    return yychar;}/** * Fills Lucene token with the current token text. */final void getText(Token t) {  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);}  /**   * Creates a new scanner   * There is also a java.io.InputStream version of this constructor.   *   * @param   in  the java.io.Reader to read input from.   */  StandardTokenizerImpl(java.io.Reader in) {    this.zzReader = in;  }  /**   * Creates a new scanner.   * There is also java.io.Reader version of this constructor.   *   * @param   in  the java.io.Inputstream to read input from.   */  StandardTokenizerImpl(java.io.InputStream in) {    this(new java.io.InputStreamReader(in));  }  /**    * Unpacks the compressed character translation table.   *   * @param packed   the packed character translation table   * @return         the unpacked character translation table   */  private static char [] zzUnpackCMap(String packed) {    char [] map = new char[0x10000];    int i = 0;  /* index in packed string  */    int j = 0;  /* index in unpacked array */    while (i < 1154) {      int  count = packed.charAt(i++);      char value = packed.charAt(i++);      do map[j++] = value; while (--count > 0);    }    return map;  }  /**   * Refills the input buffer.   *   * @return      <code>false</code>, iff there was new input.   *    * @exception   java.io.IOException  if any I/O-Error occurs   */  private boolean zzRefill() throws java.io.IOException {    /* first: make room (if you can) */    if (zzStartRead > 0) {      System.arraycopy(zzBuffer, zzStartRead,                       zzBuffer, 0,                       zzEndRead-zzStartRead);      /* translate stored positions */      zzEndRead-= zzStartRead;      zzCurrentPos-= zzStartRead;      zzMarkedPos-= zzStartRead;      zzPushbackPos-= zzStartRead;      zzStartRead = 0;    }    /* is the buffer big enough? */    if (zzCurrentPos >= zzBuffer.length) {      /* if not: blow it up */      char newBuffer[] = new char[zzCurrentPos*2];      System.arraycopy(zzBuffer, 0, newBuffer, 0, zzBuffer.length);      zzBuffer = newBuffer;    }    /* finally: fill the buffer with new input */    int numRead = zzReader.read(zzBuffer, zzEndRead,                                            zzBuffer.length-zzEndRead);    if (numRead < 0) {      return true;    }    else {      zzEndRead+= numRead;      return false;    }  }      /**   * Closes the input stream.   */  public final void yyclose() throws java.io.IOException {    zzAtEOF = true;            /* indicate end of file */    zzEndRead = zzStartRead;  /* invalidate buffer    */    if (zzReader != null)      zzReader.close();  }  /**   * Resets the scanner to read from a new input stream.   * Does not close the old reader.   *   * All internal variables are reset, the old input stream    * <b>cannot</b> be reused (internal buffer is discarded and lost).   * Lexical state is set to <tt>ZZ_INITIAL</tt>.   *   * @param reader   the new input stream    */  public final void yyreset(java.io.Reader reader) {    zzReader = reader;    zzAtBOL  = true;    zzAtEOF  = false;    zzEndRead = zzStartRead = 0;    zzCurrentPos = zzMarkedPos = zzPushbackPos = 0;    yyline = yychar = yycolumn = 0;    zzLexicalState = YYINITIAL;  }  /**   * Returns the current lexical state.   */  public final int yystate() {    return zzLexicalState;  }  /**   * Enters a new lexical state   *   * @param newState the new lexical state   */  public final void yybegin(int newState) {    zzLexicalState = newState;  }  /**   * Returns the text matched by the current regular expression.   */  public final String yytext() {    return new String( zzBuffer, zzStartRead, zzMarkedPos-zzStartRead );  }  /**   * Returns the character at position <tt>pos</tt> from the    * matched text.    *    * It is equivalent to yytext().charAt(pos), but faster   *   * @param pos the position of the character to fetch.    *            A value from 0 to yylength()-1.   *   * @return the character at position pos   */  public final char yycharat(int pos) {    return zzBuffer[zzStartRead+pos];  }  /**   * Returns the length of the matched text region.   */  public final int yylength() {    return zzMarkedPos-zzStartRead;  }  /**   * Reports an error that occured while scanning.   *   * In a wellformed scanner (no or only correct usage of    * yypushback(int) and a match-all fallback rule) this method    * will only be called with things that "Can't Possibly Happen".   * If this method is called, something is seriously wrong   * (e.g. a JFlex bug producing a faulty scanner etc.).   *   * Usual syntax/scanner level error handling should be done   * in error fallback rules.   *   * @param   errorCode  the code of the errormessage to display   */  private void zzScanError(int errorCode) {    String message;    try {      message = ZZ_ERROR_MSG[errorCode];    }    catch (ArrayIndexOutOfBoundsException e) {      message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];    }    throw new Error(message);  }   /**   * Pushes the specified amount of characters back into the input stream.   *   * They will be read again by then next call of the scanning method   *   * @param number  the number of characters to be read again.   *                This number must not be greater than yylength()!   */  public void yypushback(int number)  {    if ( number > yylength() )      zzScanError(ZZ_PUSHBACK_2BIG);    zzMarkedPos -= number;  }  /**   * Resumes scanning until the next regular expression is matched,   * the end of input is encountered or an I/O-Error occurs.   *   * @return      the next token   * @exception   java.io.IOException  if any I/O-Error occurs   */  public int getNextToken() throws java.io.IOException {    int zzInput;    int zzAction;    // cached fields:    int zzCurrentPosL;    int zzMarkedPosL;    int zzEndReadL = zzEndRead;    char [] zzBufferL = zzBuffer;    char [] zzCMapL = ZZ_CMAP;    int [] zzTransL = ZZ_TRANS;    int [] zzRowMapL = ZZ_ROWMAP;    int [] zzAttrL = ZZ_ATTRIBUTE;    while (true) {      zzMarkedPosL = zzMarkedPos;      yychar+= zzMarkedPosL-zzStartRead;      zzAction = -1;      zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;        zzState = zzLexicalState;      zzForAction: {        while (true) {              if (zzCurrentPosL < zzEndReadL)            zzInput = zzBufferL[zzCurrentPosL++];          else if (zzAtEOF) {            zzInput = YYEOF;            break zzForAction;          }          else {            // store back cached positions            zzCurrentPos  = zzCurrentPosL;            zzMarkedPos   = zzMarkedPosL;            boolean eof = zzRefill();            // get translated positions and possibly new buffer            zzCurrentPosL  = zzCurrentPos;            zzMarkedPosL   = zzMarkedPos;            zzBufferL      = zzBuffer;            zzEndReadL     = zzEndRead;            if (eof) {              zzInput = YYEOF;              break zzForAction;            }            else {              zzInput = zzBufferL[zzCurrentPosL++];            }          }          int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];          if (zzNext == -1) break zzForAction;          zzState = zzNext;          int zzAttributes = zzAttrL[zzState];          if ( (zzAttributes & 1) == 1 ) {            zzAction = zzState;            zzMarkedPosL = zzCurrentPosL;            if ( (zzAttributes & 8) == 8 ) break zzForAction;          }        }      }      // store back cached position      zzMarkedPos = zzMarkedPosL;      switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {        case 4:           { return HOST;          }        case 11: break;        case 9:           { return ACRONYM;          }        case 12: break;        case 8:           { return ACRONYM_DEP;          }        case 13: break;        case 1:           { /* ignore */          }        case 14: break;        case 5:           { return NUM;          }        case 15: break;        case 3:           { return CJ;          }        case 16: break;        case 2:           { return ALPHANUM;          }        case 17: break;        case 7:           { return COMPANY;          }        case 18: break;        case 6:           { return APOSTROPHE;          }        case 19: break;        case 10:           { return EMAIL;          }        case 20: break;        default:           if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {            zzAtEOF = true;            return YYEOF;          }           else {            zzScanError(ZZ_NO_MATCH);          }      }    }  }}