?? cdevidesentence.cpp
字號:
// CDevideSentence.cpp : Defines the entry point for the console application.
//
#include "stdafx.h"
#include "string.h"
#include "CDevideSentence.h"
#include "usedll.h"
///////////////////CDevideSentence////////////////////////////////////////////////////////
CDevideSentence::CDevideSentence()
{
szSource=NULL;
szResult=NULL;
this->bOccuredError=0;
}
CDevideSentence::~CDevideSentence()
{
if(szSource) delete szSource;
if(szResult) delete szResult;
}
unsigned char* CDevideSentence::getResult()
{
return this->szResult;
}
void CDevideSentence::inputSentence(unsigned char *para_szSentence)
{
if(szSource) delete szSource;
if(szResult) delete szResult;
int strLen=strlen((char*)para_szSentence);
if(strLen>0)
{
szSource=new unsigned char[strLen+1];
strcpy((char*)szSource,(char*)para_szSentence);
szResult=new unsigned char[strLen*2];
}
if(szSource==NULL||szResult==NULL) this->bOccuredError=1;//分配內(nèi)存出錯
}
int CDevideSentence::isChineseChr(unsigned char *para_acChr)
{
int rtCode=0;
if((para_acChr[0]>=0xb0&¶_acChr[0]<0xf8&¶_acChr[1]>0xa0&¶_acChr[1]<0xff)/*GB2312漢字*/
||(para_acChr[0]>=0x81&¶_acChr[0]<0xff&¶_acChr[1]>=0x40&¶_acChr[1]<=0xa0)/*擴(kuò)充漢字*/)
{
rtCode=1;
}
else
{
rtCode=0;
}
return rtCode;
}
int CDevideSentence::devideSentence(unsigned char *para_szSentence,unsigned char para_cSeparator)
{
CDevideWordSeg cDevideWordSeg;
this->inputSentence(para_szSentence);
this->chrDevideCharacter=para_cSeparator;
if(this->bOccuredError==1)
{
return -1;
}
//對句子進(jìn)行分詞。
//先將句子分隔成字段（全由中文組成的串），對各串分詞
int nSourcePos=0;//讀取原串的位置
int nResultPos=0;//寫入結(jié)果串的位置。
int nScanPos;//當(dāng)前掃描的位置
nScanPos=nSourcePos;
while(this->szSource[nScanPos]!=0)
{//字串尚未掃描結(jié)束
int rtval=isChineseChr((unsigned char*)szSource+nScanPos);
if(rtval==0)
{//遇到漢字外的字符
unsigned char storecurrentchr=szSource[nScanPos];//保留當(dāng)前掃描到的字符
int isWide=0;//是否全角，默認(rèn)不是
if(storecurrentchr>=0x80)
{//是全角
isWide=1;
}
szSource[nScanPos]=0;
if(0!=cDevideWordSeg.devideWordSeg(szSource+nSourcePos/*待切分中文字段*/,szResult+nResultPos/*輸出的帶分隔符的以0結(jié)尾的結(jié)果串*/,chrDevideCharacter/*分隔符*/))
{
return -1;
}
int endposofresult=nResultPos;
while(szResult[endposofresult]!=0) endposofresult++;
if(strlen((char*)szSource+nSourcePos)!=0)//如果長度為0則表示連續(xù)出現(xiàn)兩個非中文字符，開頭不能再加分隔符
{
szResult[endposofresult++]=chrDevideCharacter;
}
szResult[endposofresult++]=storecurrentchr;
if(isWide==1)
{
szResult[endposofresult++]=szSource[++nScanPos];
}
szResult[endposofresult++]=chrDevideCharacter;
nResultPos=endposofresult;
nScanPos++;
nSourcePos=nScanPos;
}
else
{//是漢字，繼續(xù)掃描下去
nScanPos+=2;
}
}
cDevideWordSeg.devideWordSeg(szSource+nSourcePos/*待切分中文字段*/,szResult+nResultPos/*輸出的帶分隔符的以0結(jié)尾的結(jié)果串*/,chrDevideCharacter/*分隔符*/);
return 0;
}
//////////////////////////CDevideWordSeg/////////////////////////////////////////////////////////
void CDevideWordSeg::buildDifferentMeaningInfo()
{
//int getWord(char*/*要找的詞*/,unsigned short*/*返回的詞的詞性*/,int*/*返回的詞的頻率*/);//取得給定詞的信息，返回詞的位置（>0）：成功，－1：失敗
int length=strlen((char*)source)/2;//取得源串的長度,(中文的長度)
int startpos=0;//開始掃描的位置
for(startpos=0;startpos<length;startpos++)
{
int j=10;
if((length-startpos)<j) j=length-startpos;
for(;j>0;j--)
{
if(j!=1)
{
unsigned char storechar=*(source+startpos*2+j*2);//暫存
*(source+startpos*2+j*2)=0;
unsigned short attr;
int frequancy;
getWord((char*)source+startpos*2,&attr,&frequancy);
if(frequancy>0)
{//頻率大于0，肯定為詞
pTable[0][startpos]=j-1+startpos;
*(source+startpos*2+j*2)=storechar;//恢復(fù)
break;
}
*(source+startpos*2+j*2)=storechar;//恢復(fù)
}
else
{
pTable[0][startpos]=startpos;
}
}
}
/* for(startpos=0;startpos<length;startpos++)
{
if(startpos==pTable[0][startpos])
{
pTable[0][startpos]=-1;
startpos++;
}
int maxpos=pTable[0][startpos];
for(int i=startpos;i<=maxpos;i++)
{
if(pTable[0][startpos]>maxpos)
{
maxpos=pTable[0][startpos];
}
}
if(startpos==maxpos)
{
pTable[0][startpos]=-1;
}
startpos=maxpos;
}
*/
}
int CDevideWordSeg::getSumFrequancy(unsigned char *para_acSource,int para_nLen)//取字段的總詞頻
{
int rtCode=0;
unsigned short attr;
int frequancy;
for(int i=0;i<para_nLen;i++)
{
for(int j=i;j<para_nLen;j++)
{
unsigned char storechar=*(para_acSource+(j+1)*2);
*(para_acSource+(j+1)*2)=0;
getWord((char*)para_acSource+i*2,&attr,&frequancy);
rtCode+=frequancy;
*(para_acSource+(j+1)*2)=storechar;
}
}
return rtCode;
}
void CDevideWordSeg::devideDifferentMeaningWordSeg(int para_nStart,int para_nEnd,int para_nStartpos,int para_nEndpos)//切分奇異字段
{//切分結(jié)果暫存在pTable[10]中，然后計算切分概率后再插入1－9行
unsigned short attr;
int frequancy;
int startpos=para_nStartpos;
int endpos=para_nEndpos;
if(startpos>endpos)
{//結(jié)束一種切分，開始進(jìn)行計算概率并插入適當(dāng)位置。
// para_nStart,para_nEnd;
int tempfrequancy=0;//暫存詞的概率
tempfrequancy=pTable[10][para_nStart];
for(int i=para_nStart;i<=para_nEnd+1;i++)
{
if(i==para_nEnd+1)
{
int fenzi=pTable[10][i-1];
if(pTable[10][i-1]==0)
{
fenzi=1;
}
if(this->afPropability[10]==0)
{
this->afPropability[10]=(double)fenzi/(double)pTable[11][i-1];
}
else
{
this->afPropability[10]*=(double)fenzi/(double)pTable[11][i-1];
}
break;
}
if(pTable[10][i]!=tempfrequancy)
{//
tempfrequancy=pTable[10][i];
int fenzi=pTable[10][i-1];
if(pTable[10][i-1]==0)
{
fenzi=1;
}
if(this->afPropability[10]==0)
{
this->afPropability[10]=(double)fenzi/(double)pTable[11][i-1];
}
else
{
this->afPropability[10]*=(double)fenzi/(double)pTable[11][i-1];
}
}
}
//插入1－9中適當(dāng)位置
int pos=1;
while(pos<9&&this->afPropability[10]<this->afPropability[pos]) pos++;
//now this->afPropability[10]>=afPropability[pos];
if(this->afPropability[1]>0)
{
for(int kk=8;kk>=pos;kk--)
{
memcpy(pTable[kk+1]+para_nStart,pTable[kk]+para_nStart,sizeof(int)*(para_nEnd-para_nStart+1));
this->afPropability[kk+1]=this->afPropability[kk];
}
}
memcpy(pTable[pos]+para_nStart,pTable[10]+para_nStart,sizeof(int)*(para_nEnd-para_nStart+1));
this->afPropability[pos]=this->afPropability[10];
this->afPropability[10]=0;
}
for(int i=startpos;i<=endpos;i++)
{
unsigned char storechar=*(source+(i+1)*2);
*(source+(i+1)*2)=0;
getWord((char*)source+para_nStartpos*2,&attr,&frequancy);
int rtval=frequancy;
*(source+(i+1)*2)=storechar;
if(startpos>0)
{
if(pTable[10][startpos-1]==rtval)
{
if(rtval>0)
{
rtval--;
}
else
{
rtval++;
}
}
}
for(int k=startpos;k<=i;k++)
{
if(rtval<2)
{
if(k>0)
{
rtval=pTable[10][k-1]+1;
rtval%=2;
pTable[10][k]=rtval;
}
else
{
pTable[10][k]=0;
}
}
else
{
pTable[10][k]=rtval;
}
}
devideDifferentMeaningWordSeg(para_nStart,para_nEnd,i+1,para_nEnd);
}
}
int CDevideWordSeg::devideWordSeg(unsigned char *para_szSource,unsigned char *para_szResult,unsigned char para_cSeparator)
{
int rtCode=0;
source=para_szSource;//待切分串，以0結(jié)尾
result=para_szResult;//用來存儲切分結(jié)果，結(jié)果串必須以0結(jié)尾
unsigned char separator=para_cSeparator;//分隔符號
int length=strlen((char*)source)/2;//取得源串的長度,（中文的長度）
while(length>=this->nMaxLen)
{
this->addMem();
if(this->bOccuredError==1)
{//分配內(nèi)存出錯
return -1;
}
}
int startpos=0;//開始掃描的位置
if(bOccuredError==0)
{
this->buildDifferentMeaningInfo();
for(startpos=0;startpos<length;startpos++)
{
int maxpos=pTable[0][startpos];
for(int i=startpos;i<=maxpos;i++)
{
if(pTable[0][i]>maxpos)
{
maxpos=pTable[0][i];
}
}
//取奇異字段（source+startpos×2,source+maxpos×2）的總詞頻
int sumfrequancy=this->getSumFrequancy(source+startpos*2,maxpos-startpos+1);
for(int k=startpos;k<=maxpos;k++)
{//將奇異字段信息填入pTable第11行
pTable[11][k]=sumfrequancy;
}
this->devideDifferentMeaningWordSeg(startpos,maxpos,startpos,maxpos);
for(k=0;k<12;k++) this->afPropability[k]=0;//初始化已計算的歧義段概率
startpos=maxpos;
}
}
else
{
rtCode=-1;
}
//到此pTable[1]-pTable[9]存儲了分隔方式。
int resultpos=0;//輸出串輸入地址
int tempfrequancy=pTable[1][0];
for(int i=0;i<length;i++)
{
if(pTable[1][i]!=tempfrequancy)
{//
tempfrequancy=pTable[1][i];
result[resultpos++]=separator;
result[resultpos++]=source[i*2];
result[resultpos++]=source[i*2+1];
}
else
{
result[resultpos++]=source[i*2];
result[resultpos++]=source[i*2+1];
}
}
result[resultpos++]=0;
return rtCode;
}
CDevideWordSeg::CDevideWordSeg()
{
//int getWord(char*/*要找的詞*/,unsigned short*/*返回的詞的詞性*/,int*/*返回的詞的頻率*/);//取得給定詞的信息，返回詞的位置（>0）：成功，－1：失敗
bOccuredError=0;
nMaxLen=200;
for(int i=0;i<12;i++)
{
pTable[i]=new int[nMaxLen];
if(pTable[i]==NULL)
{
bOccuredError=1;
}
this->afPropability[i]=0;
}
hInstance=LoadLibrary("wordlibaccess.dll");//加載訪問詞庫的動態(tài)庫
if(hInstance)
{//加載成功
loadLib=(loadLib_dll)GetProcAddress(hInstance,"loadLib_dll");//取裝入詞庫的函數(shù)
if(loadLib)
{//取函數(shù)成功
if(loadLib("wordlib.bin")==-1)
{//加載詞庫失敗
this->bOccuredError=1;
}
else
{//詞庫加載成功
getWord=(getWord_dll)GetProcAddress(hInstance,"getWord_dll");//取取詞頻的函數(shù)
if(getWord==NULL)
{//取函數(shù)失敗
this->bOccuredError=1;
}
}
}
}
else
{
this->bOccuredError=1;
}
}
CDevideWordSeg::~CDevideWordSeg()
{
for(int i=0;i<12;i++)
{
if(pTable[i])
{
delete pTable[i];
}
}
if(hInstance)
{
FreeLibrary(hInstance);
hInstance=NULL;
}
}
void CDevideWordSeg::addMem()
{
for(int i=0;i<11;i++)
{
if(pTable[i])
{
delete pTable[i];
}
pTable[i]=new int[nMaxLen*2];
if(pTable[i]==NULL)
{
bOccuredError=1;
}
}
nMaxLen*=2;
}
////////////////////////////////////////////////////////////////////////////////////////////
?? 快捷鍵說明
復(fù)制代碼
Ctrl + C
搜索代碼
Ctrl + F
全屏模式
F11
切換主題
Ctrl + Shift + D
顯示快捷鍵
?
增大字號
Ctrl + =
減小字號
Ctrl + -