/*** 2006 July 10**** The author disclaims copyright to this source code.***************************************************************************** Defines the interface to tokenizers used by fulltext-search.  There** are three basic components:**** sqlite3_tokenizer_module is a singleton defining the tokenizer** interface functions.  This is essentially the class structure for** tokenizers.**** sqlite3_tokenizer is used to define a particular tokenizer, perhaps** including customization information defined at creation time.**** sqlite3_tokenizer_cursor is generated by a tokenizer to generate** tokens from a particular input.*/#ifndef _FTS1_TOKENIZER_H_#define _FTS1_TOKENIZER_H_/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.** If tokenizers are to be allowed to call sqlite3_*() functions, then** we will need a way to register the API consistently.*/#include "sqlite3.h"/*** Structures used by the tokenizer interface.*/typedef struct sqlite3_tokenizer sqlite3_tokenizer;typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;struct sqlite3_tokenizer_module {  int iVersion;                  /* currently 0 */  /*  ** Create and destroy a tokenizer.  argc/argv are passed down from  ** the fulltext virtual table creation to allow customization.  */  int (*xCreate)(int argc, const char *const*argv,                 sqlite3_tokenizer **ppTokenizer);  int (*xDestroy)(sqlite3_tokenizer *pTokenizer);  /*  ** Tokenize a particular input.  Call xOpen() to prepare to  ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then  ** xClose() to free any internal state.  The pInput passed to  ** xOpen() must exist until the cursor is closed.  The ppToken  ** result from xNext() is only valid until the next call to xNext()  ** or until xClose() is called.  */  /* TODO(shess) current implementation requires pInput to be  ** nul-terminated.  This should either be fixed, or pInput/nBytes  ** should be converted to zInput.  */  int (*xOpen)(sqlite3_tokenizer *pTokenizer,               const char *pInput, int nBytes,               sqlite3_tokenizer_cursor **ppCursor);  int (*xClose)(sqlite3_tokenizer_cursor *pCursor);  int (*xNext)(sqlite3_tokenizer_cursor *pCursor,               const char **ppToken, int *pnBytes,               int *piStartOffset, int *piEndOffset, int *piPosition);};struct sqlite3_tokenizer {  const sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */  /* Tokenizer implementations will typically add additional fields */};struct sqlite3_tokenizer_cursor {  sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */  /* Tokenizer implementations will typically add additional fields */};/*** Get the module for a tokenizer which generates tokens based on a** set of non-token characters.  The default is to break tokens at any** non-alnum character, though the set of delimiters can also be** specified by the first argv argument to xCreate().*//* TODO(shess) This doesn't belong here.  Need some sort of** registration process.*/void sqlite3Fts1SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);void sqlite3Fts1PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);#endif /* _FTS1_TOKENIZER_H_ */