/******************************************
	http://www.sqlet.com
	mail:199909@gmail.com
	
	中文分詞測試版	
	author:linfj
	詞典文件:sqlet.dict
*******************************************/
#include <stdio.h>
#include <string.h>
#include <malloc.h>

#define BOOL				char
#define bool				BOOL
#define TRUE				1
#define FALSE				0
#define MAX_CWORD_LEN			10				//最長的詞
#define MAX_SWORD_LEN			256				//最長的句子
#define	MAX_CDIM			90

//不進行索引的單詞
char *arrayEnglishStop[] = {
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", 
"1", "2", "3", "4", "5", "6", "7", "8", "9", "0",  
"about", "above", "after", "again", "all", "also", "am", "an", "and", "any", "are", "as", "at", 
"back", "be", "been", "before", "behind", "being", "below", "but", "by", 
"can", "click", "do", "does", "done", "each", "else", "etc", "ever", "every", 
"few", "for", "from", "generally", "get", "go", "gone", "has", "have", "hello", "here", "how", 
"if", "in", "into", "is", "just", "keep", "later", "let", "like", "lot", "lots", "made", 
"make", "makes", "many", "may", "me", "more", "most", "much", "must", "my", "need", "no", "not", 
"now", "of", "often", "on", "only", "or", "other", "others", "our", "out", "over", "please", "put", 
"so", "some", "such", "than", "that", "the", "their", "them", "then", "there", "these", "they", 
"this", "try", "to", "up", "us", "very", "want", "was", "we", "well", "what", "when", "where", 
"which", "why", "will", "with", "within", "you", "your", "yourself"
};

//詞典索引時,字或詞不需要索引
char *arrayChineseStop[] = {
"的","嗎","么","啊","說","對","在","和","是",
"被","最","所","那","這","有","將","會","與",
"於","于","他","她","它","您","為","歡迎"
};

//全角的ASCII,要全部轉成半角英文字符,以后還要加入其它的符號如,.;/|等
//區碼為163的都要轉成ascii
//163	！ ＂ ＃ ￥ ％ ＆ ＇ （ ） ＊ ＋ ， － ． ／ ０ １ ２ ３ ４ ５ ６ ７ ８ ９ ： ； ＜ ＝ ＞ ？ ＠ Ａ Ｂ Ｃ Ｄ Ｅ Ｆ Ｇ Ｈ Ｉ Ｊ Ｋ Ｌ Ｍ Ｎ Ｏ Ｐ Ｑ Ｒ Ｓ Ｔ Ｕ Ｖ Ｗ Ｘ Ｙ Ｚ ［ ＼ ］ ＾ ＿ ｀ ａ ｂ ｃ ｄ ｅ ｆ ｇ ｈ ｉ ｊ ｋ ｌ ｍ ｎ ｏ ｐ ｑ ｒ ｓ ｔ ｕ ｖ ｗ ｘ ｙ ｚ ｛ ｜ ｝ ￣ 
//		!	  #  $  %  &  '   (  )  % +  ,  -  .  /   0   1  2 		
unsigned char *arrayWideAscii[] = {
"ａ","ｂ","ｃ","ｄ","ｅ","ｆ","ｇ","ｈ","ｉ","ｊ","ｋ","ｌ","ｍ","ｎ","ｏ","ｐ","ｑ","ｒ","ｓ","ｔ","ｕ","ｖ","ｗ","ｘ","ｙ","ｚ","．",
"Ａ","Ｂ","Ｃ","Ｄ","Ｅ","Ｆ","Ｇ","Ｈ","Ｉ","Ｊ","Ｋ","Ｌ","Ｍ","Ｎ","Ｏ","Ｐ","Ｑ","Ｒ","Ｓ","Ｔ","Ｕ","Ｖ","Ｗ","Ｘ","Ｙ","Ｚ","－"
};

//標點符號及漢字的標點符號,注意 + - " 這三個符號，因為在搜索的時候需要通過他們進行異或等條件判斷

char arrayAsciiSymbol[] ={
'!','\\','*','(',')','-','_','+','=','{','}','[',']',':',';','\'','\"',',','<','>','.','?','/','|','@','#','$','%','^','&'
};

//BIG5與GB對照,把所有big5轉成gb后進行處理
//$arrayBig5ToG = array ();
//UTF8的轉換

//漢字詞典
typedef struct _WORD_NODE
{
	char	strWord[MAX_CWORD_LEN+1];
	// todo ,可以增加 兩個字，三個字，四個字，五個字的數組，這樣查起來更快
	struct  _WORD_NODE	*nextWord;
}WORD_NODE;
//分詞結果
typedef struct _SEG_NODE
{
	char	strWord[MAX_CWORD_LEN+1];
	struct	_SEG_NODE	*nextWord;
}SEG_NODE;

struct _CH_DICT 
{
	WORD_NODE	*lstWord;
}CH_DICT[MAX_CDIM][MAX_CDIM];

struct _SEG_LIST 
{
	SEG_NODE	*lstWord;
}SEG_LIST[MAX_CDIM][MAX_CDIM];

/*同義詞典
	咖啡館,咖啡屋
	神六，神舟六號
synonymous
*/
char  *strTrim(char str[])
{
  int firstchar=0;
  int endpos=0;
  int i;
  int firstpos=0;

  for(i=0;str[i]!='\0';i++)
  {
    if(str[i]==' ' || str[i] == '\r' || str [i] == '\n' || str [i]=='\t')
	{
      if(firstchar==0) firstpos++;
    }
    else
	{
      endpos=i;
      firstchar=1;
    }
  }

  for(i=firstpos;i<=endpos;i++)
    str[i-firstpos]=str[i];
  str[i-firstpos]='\0';

  return str;
}

int addDictWord(char *strWord , int len)
{
	unsigned char firstChar,lastChar;
	WORD_NODE* curLst;
	
	WORD_NODE* newWord ,*curTmp ;
	
	firstChar = strWord[0] ;
	lastChar  = strWord[len-1];
	
	if (firstChar < 161 || lastChar < 161 )	//非漢字或是漢字的全角符號
		  return -1;
	
	newWord = (WORD_NODE*)malloc(sizeof(WORD_NODE));
	if ( newWord == NULL)
		return -1;
	strcpy(newWord->strWord,strWord);
	newWord->nextWord = NULL;
	
	firstChar -= 161 ;
	lastChar  -= 161 ;
	curLst = CH_DICT[firstChar][lastChar].lstWord;

	if( curLst == NULL)		//reinit list;
	{
		CH_DICT[firstChar][lastChar].lstWord = newWord ;

		return 0;
	}
	curTmp = curLst ;
	while(curTmp -> nextWord != NULL )
	{
		curTmp = curTmp->nextWord;
	}

	curTmp -> nextWord = newWord ;
	
	return 0;
}

int addSegWord(unsigned char *strWord , int len)
{
	unsigned char firstChar,lastChar;
	SEG_NODE* curLst;
	
	SEG_NODE* newWord ,*curTmp ;
	
	firstChar = strWord[0] ;
	lastChar  = strWord[len-1];

	//查看是否已經存在	
	firstChar %= MAX_CDIM ;
	lastChar  %= MAX_CDIM ;
	curLst = SEG_LIST[firstChar][lastChar].lstWord;

	curTmp = curLst ;
	while(curTmp != NULL )
	{
//		if ( strcasecmp(curTmp->strWord,(char *)strWord) == 0)
		if ( strcmp(curTmp->strWord,(char *)strWord) == 0)
			 return 0;		//已經存在
		curTmp = curTmp->nextWord;
	}

	newWord = (SEG_NODE*)malloc(sizeof(SEG_NODE));
	if ( newWord == NULL)
		return -1;
	strcpy(newWord->strWord,(char *)strWord);
	newWord->nextWord = NULL;
	
	if( curLst == NULL)		//reinit list;
	{
		SEG_LIST[firstChar][lastChar].lstWord = newWord ;

		return 0;
	}
	curTmp = curLst ;
	while(curTmp -> nextWord != NULL )
	{
		curTmp = curTmp->nextWord;
	}

	curTmp -> nextWord = newWord ;
	
	return 0;
}

int freeDict()
{
	int i ,j ;
	WORD_NODE *curLst,*curTmp ,*tmp;
	
	for ( i = 0 ; i < MAX_CDIM ; i ++ )
	  for ( j = 0 ; j < MAX_CDIM ; j ++ )
	  {
			curLst = CH_DICT[i][j].lstWord;
			curTmp = curLst;
			while ( curTmp != NULL ) 
			{
				tmp = curTmp ;
		//		printf("%s|",curTmp->strWord);
				curTmp = curTmp -> nextWord ;
//				if ( curTmp == NULL)
//					printf("\n");

				free(tmp);
				tmp = ( WORD_NODE *)NULL;
			}
			CH_DICT[i][j].lstWord = (WORD_NODE *)NULL;
	 }
	  return TRUE;
}

int freeSeg()
{
	int i ,j ;
	SEG_NODE *curLst,*curTmp ,*tmp;
	
	for ( i = 0 ; i < MAX_CDIM ; i ++ )
	  for ( j = 0 ; j < MAX_CDIM ; j ++ )
	  {
			curLst = SEG_LIST[i][j].lstWord;
			curTmp = curLst;
			while ( curTmp != NULL )
			{
				tmp = curTmp ;
				printf("%s|",curTmp->strWord);
				curTmp = curTmp -> nextWord ;
				if ( curTmp == NULL)
					printf("\n");

				free(tmp);
				tmp = ( SEG_NODE *)NULL;
			}
			SEG_LIST[i][j].lstWord = (SEG_NODE *)NULL;
	 }
	  return TRUE;
}

BOOL searchWord( unsigned char *strWord,int len )
{
	WORD_NODE *curLst,*curTmp;
	unsigned char firstChar,lastChar;
	
	firstChar = strWord[0] ;
	lastChar  = strWord[len-1];

	firstChar -= 161 ;
	lastChar  -= 161 ;
	curLst = CH_DICT[firstChar][lastChar].lstWord;

	curTmp = curLst;
	while ( curTmp != NULL ) 
	{
		if ( strcmp((char *)strWord,curTmp->strWord) == 0)
				return TRUE;
		curTmp = curTmp -> nextWord ;
	}
	return FALSE;
}

int segWord ( unsigned char *strText  , int iWordLen , BOOL bChinese )
{
	int i = 0 ,j = 0 , k = 0 , l = 0;
	unsigned char strChar[MAX_CWORD_LEN+1],strChar1[5],strChar2[5],strChar3[7];	
	BOOL	bFound = FALSE;
	
	i = iWordLen  ; 
	if ( FALSE == bChinese )
	{		//英文	
		//檢查 是否在stop數組里
		addSegWord(strText,iWordLen);
		return 0;	
	}