// Dictionary.cpp: implementation of the Dictionary class.
#include "afxmt.h"
#include "Dictionary.h"
#include "TextMerger.h"
#include <set>
#include <map>
#include "EnglishDictionary.h"
#include "PreprocFuncs.h"

//#ifdef _DEBUG
//#undef THIS_FILE
//static char THIS_FILE[]=__FILE__;
//#define new DEBUG_NEW
//#endif

//////////////////////////////////////////////////////////////////////
// Construction/Destruction
//////////////////////////////////////////////////////////////////////

#define WHTSPC(c) ((c) == ' ' || (c) == '\t')
#define ISALETTER(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
#define NOTALETTER(c) ( ! ISALETTER(c) )
#define ISUPPERCASE(c) ((c) >= 'A' && (c) <= 'Z')
#define ISEOL(c) ((c) == '\n' || (c) == 'r' || (c) == '\0')
#define ISDIGIT(c)  ((c) >= '0' && (c) <= '9')

CTextMerger m_textMerger;

CCriticalSection g_ccDictionaryInit;

static int CountMeaningFullChars(const WCHAR* text)
{
	int count = 0;

	while (*text != '\0')
	{
		if ( (*text != '\r') && (*text != '\n') )
			++count;
		++text;
	}

	return count;
}

static void RemoveEndsOfLines(char* text)
{
	if (!text)
		return;

	for (char *p = text; *p != '\0'; ++p)
	{
		if (*p != '\n')
		{
			if (*p == '-' && p[1] == '\r')
				p += 2;

			*text++ = (*p == '\r') ? ' ' : *p;
		}
	}

	*text = '\0';
}

static  void RemoveEndsOfLines(wchar_t* text)
{
	if (!text)
		return;

	for (wchar_t* p = text; *p != L'\0'; ++p)
	{
		if (*p != L'\n')
		{
			if (*p == L'-' && p[1] == L'\r')
				p += 2;

			*text++ = (*p == L'\r') ? L' ' : *p;
		}
	}

	*text = L'\0';
}

WCHAR *GetNextWord(WCHAR *pWords){
	while(*pWords != '\n')++pWords;
	if(pWords[1] == '#'){
		return NULL;
	}
	return pWords + 1;
}

int CountWords(const WCHAR *pDict, int nSize) {
	int nCount = 0;
	for(const WCHAR *pDEnd = pDict + nSize; pDict < pDEnd; ++pDict){
		if(*pDict == '\n'){
			++nCount;
		}
	}
	return nCount;
}

int Dictionary::ToLower(const WCHAR *pSrc, WCHAR *pDest) const {
	int nCount = 0;
	while((*pDest = ToLower(*pSrc)) != 0){
		if(*pDest != '\'') ++pDest;
		++pSrc;
		++nCount;
	}
	*pDest = '\0';
	return nCount;
}

void Dictionary::ToLower(wstring & s) const {
	int nCount = 0;
	for(wstring::iterator i = s.begin(); i != s.end(); ++i) {
		*i = ToLower(*i);
	}
}

int Dictionary::GetWordLength(const WCHAR *pWord)const{
	int nCount = 0;
	while(ToLower(*pWord) != 0){
		++pWord;
		++nCount;
	}
	return nCount;
}

int Dictionary::CompareWords(const WCHAR *w0, const WCHAR *w1)const{
	static WCHAR word0[MAX_PATH], word1[MAX_PATH];
	int nL0 = ToLower(w0, word0);
	int nL1 = ToLower(w1, word1);
	int nL = min(nL0, nL1);
	for(int i=0; i<nL; ++i){
		int nIndx0 = GettLetterIndx(word0[i]);
		int nIndx1 = GettLetterIndx(word1[i]);
		if(nIndx0 < nIndx1){
			return -1;
		}
		if(nIndx0 > nIndx1){
			return 1;
		}
	}
	if(nL0 > nL1)return 2;
	if(nL0 < nL1)return -2;
	return 0;
}

void Dictionary::GetFirst3Letters(const WCHAR *pCurrWord, WCHAR *p3Letters) const {
	while(*pCurrWord == '\''){++pCurrWord;}
	p3Letters[0] = *pCurrWord++;
	while(*pCurrWord == '\''){++pCurrWord;}
	p3Letters[1] = *pCurrWord++;
	while(*pCurrWord == '\''){++pCurrWord;}
	p3Letters[2] = *pCurrWord++;
}

int Dictionary::WordLength(const WCHAR *pCurrWord) const {
	int nCount = 0;
	while(*pCurrWord != '\n' && *pCurrWord != '\r' && *pCurrWord != '/' && *pCurrWord != '#'){
		if(*pCurrWord != '\''){
			++nCount;
		}
		++pCurrWord;
	}
	return nCount;
}

Dictionary* Dictionary::mInstance = NULL;
int Dictionary::m_nLangIndx = -1;

Dictionary::Dictionary()
{
	m_pDict = NULL;
	m_pWords = NULL;
	m_pIndx = NULL;
	m_pTwoLetWords = NULL;
	m_nNUM_OF_LETTERS = 0;
}

Dictionary::~Dictionary()
{
	if(m_pDict) {delete [] m_pDict; m_pDict = NULL;}
	if(m_pWords) {delete [] m_pWords; m_pWords = NULL;}
	if(m_pIndx){delete [] m_pIndx; m_pIndx = NULL;}
	if(m_pOneLetWords){delete [] m_pOneLetWords; m_pOneLetWords = NULL;}
	if(m_pTwoLetWords){delete [] m_pTwoLetWords; m_pTwoLetWords = NULL;}
}

Dictionary* Dictionary::Instance()
{
	if (mInstance == NULL)
	{
		mInstance = SetLanguage(1); 
	}

	return mInstance;
}

BOOL Dictionary::CheckNotMatchedWord(const WCHAR *pWord) {
	int nLength = ToLower(pWord, m_pCurrWord);
	int i;
	for(i=0; !(WHTSPC(pWord[i]) || ISEOL(pWord[i])); ++i){
		if(pWord[i] <= '9' && pWord[i] >= '0')
			return TRUE;
	}
	if(nLength < 4){
		return FALSE;
	}
	BOOL bUpp = TRUE, bLow = TRUE;
	for(i=1; i<nLength; ++i){
		if(ISUPPERCASE(pWord[i])){
			bLow = FALSE;
		}
		else{
			bUpp = FALSE;
		}
	}

	if(bLow){
		return TRUE;
	}
	if(bUpp){
		if(ISUPPERCASE(pWord[0])){
			return TRUE;
		}
		return FALSE;
	}
	return FALSE;
}


BOOL  Dictionary::LookUp(const WCHAR *pWord) {
	int nLength = ToLower(pWord, m_pCurrWord);
	int nIndx = 0;
	switch(nLength){
	case 0:
		return FALSE;
		break;
	case 1:
		nIndx = GettLetterIndx(*m_pCurrWord);
		if(nIndx < 0 || nIndx >= m_nNUM_OF_LETTERS) return FALSE;
		return m_pOneLetWords[GettLetterIndx(*m_pCurrWord)];
		break;
	case 2:
		nIndx = GettLetterIndx(*m_pCurrWord) * m_nNUM_OF_LETTERS + GettLetterIndx(m_pCurrWord[1]);
		if(nIndx < 0 || nIndx >= m_nNUM_OF_LETTERS * m_nNUM_OF_LETTERS) return FALSE;
		return m_pTwoLetWords[GettLetterIndx(*m_pCurrWord) * m_nNUM_OF_LETTERS + GettLetterIndx(m_pCurrWord[1])];
		break;
	default:
		{
			int nIndx = GettLetterIndx(m_pCurrWord[0]) * m_nNUM_OF_LETTERS * m_nNUM_OF_LETTERS
				+ GettLetterIndx(m_pCurrWord[1]) * m_nNUM_OF_LETTERS
				+ GettLetterIndx(m_pCurrWord[2]);
			if(nIndx < 0 || nIndx >= m_nNUM_OF_LETTERS * m_nNUM_OF_LETTERS * m_nNUM_OF_LETTERS
				|| m_pIndx[nIndx].m_nFirst == -1){
				return FALSE;
			}
			int nFirst = m_pIndx[nIndx].m_nFirst;
			int nLast = m_pIndx[nIndx].m_nLast;
			do{
				if(nLast == nFirst){
					return CompareWords(m_pWords[nFirst], pWord) == 0;
				}

				if(nLast - nFirst == 1){
					return CompareWords(m_pWords[nFirst], pWord) == 0 ||
						CompareWords(m_pWords[nLast], pWord) == 0;
				}
				int nCIndx = (nFirst + nLast) / 2;
				int nCmp = CompareWords(m_pWords[nCIndx], pWord);
				if(nCmp == 0){
					return TRUE;
				}
				if(nCmp < 0){
					nFirst = nCIndx;
				}
				if(nCmp > 0){
					nLast = nCIndx;
				}
			}while(nFirst != nLast);
		}
	}
	return FALSE;
}

void Dictionary::AddToShortWords(const WCHAR *pCurrWord, int nLength){
	if(nLength == 1){
		m_pOneLetWords[GettLetterIndx(*pCurrWord)] = TRUE;
	}
	if(nLength == 2){
		m_pTwoLetWords[GettLetterIndx(*pCurrWord) * m_nNUM_OF_LETTERS + GettLetterIndx(pCurrWord[1])] = TRUE;
	}
}

const WCHAR *GetNextWordFromLine(const WCHAR *pLine){
	while(WHTSPC(*pLine)){++pLine;}
	return pLine;
}

int CountTildas(const WCHAR *pLine){
	int nCount = 0;
	while(*pLine != '\n' && *pLine != '\r' && *pLine != '\0'){
		if(*pLine == '~')
			++nCount;
		++pLine;
	}
	return nCount;
}

int Dictionary::FilterLine(const WCHAR *pLine, WCHAR *pNewLine) {
	int nTildas = CountTildas(pLine);
	if(nTildas * 10 > int(wcslen((wchar_t*)pLine))){
		return FALSE;
	}
	int nNoWord = 0, nWord = 0;
	const WCHAR *pL;
	BOOL bCopy = FALSE;
	while(*pLine != '\n' && *pLine != '\r' && *pLine != '\0'){
		pL = GetNextWordFromLine(pLine);
		if(pL > pLine){
			*pNewLine++ = ' ';
			pLine = pL;
		}
		if(LookUp(pLine)){
			++nWord;
			bCopy = TRUE;
		}
		else{
			if(bCopy = CheckNotMatchedWord(pLine)){
				++nWord;
			}
			else{
				++nNoWord;
			}
		}
		if(bCopy){
			while(!WHTSPC(*pLine) && *pLine != '\n' && *pLine != '\r' && *pLine != '\0'){*pNewLine++ = *pLine++;}
		}
		else{
			while(!WHTSPC(*pLine) && *pLine != '\n' && *pLine != '\r' && *pLine != '\0'){++pLine;}
		}
	}
	return (nWord + nNoWord == 0) ? 0 : (nWord << 8) / (nWord + nNoWord);
}

const WCHAR *Dictionary::CountValidWords(const WCHAR *pLine, int &nWord, int &nNoWord) {
	nNoWord = nWord = 0;
	while(*pLine != '\n' && *pLine != '\r' && *pLine != '\0'){
		pLine = GetNextWordFromLine(pLine);
		if(*pLine == '\n' || *pLine == '\r' || *pLine == '\0')break;
		if(LookUp(pLine) || CheckNotMatchedWord(pLine)){
			++nWord;
		}
		else{
			++nNoWord;
		}
		while(!WHTSPC(*pLine) && *pLine != '\n' && *pLine != '\r' && *pLine != '\0'){++pLine;}
	}
	return pLine;
}

const WCHAR *Dictionary::CountFoundWords(const WCHAR *pLine, int &nWord, int &nNoWord, int nMinWordLength) {
	nNoWord = nWord = 0;
	while(*pLine != '\n' && *pLine != '\r' && *pLine != '\0'){
		pLine = GetNextWordFromLine(pLine);
		if(*pLine == '\n' || *pLine == '\r' || *pLine == '\0')break;
		if(GetWordLength(pLine) >= nMinWordLength){
			if(LookUp(pLine)){
				++nWord;
			}
			else{
				++nNoWord;
			}
		}
		while(!WHTSPC(*pLine) && *pLine != '\n' && *pLine != '\r' && *pLine != '\0'){++pLine;}
	}
	return pLine;
}

WCHAR *RemoveWhiteSpaces(const WCHAR *pSrc){
	WCHAR *pRes = new WCHAR[wcslen((wchar_t*)pSrc) + 1], *p;
	for(p = pRes; *pSrc != '\0'; ++pSrc){
		if( !WHTSPC(*pSrc)){
			*p++ = *pSrc;
		}
	}
	*p = '\0';
	return pRes;
}

int CountRepeatedCharacters(const WCHAR *pText, const WCHAR *pCharSet){
	WCHAR *pT = RemoveWhiteSpaces(pText);
	set<WCHAR> setChar;
	const WCHAR *p;
	for(p = pCharSet; *p != '\0'; ++p){
		setChar.insert(*p);
	}
	int nCount = 0, nMaxCount = 0;
	for(p = pT; *p != '\0'; ++p){
		if(setChar.find(*p) == setChar.end()){
			nCount = 0;
		}
		else{
			++nCount;
			if(nCount > nMaxCount){
				nMaxCount = nCount;
			}
		}
	}
	delete [] pT;
	return nMaxCount;
}

bool Dictionary::CheckText(const WCHAR *pText) {
	int nWord, nNoWord;
	CountFoundWords(pText, nWord, nNoWord, 3);
	if(nWord >= 1){
		return true;
	}
	WCHAR chareset[256];
	wcscpy((wchar_t*)chareset, L"Iil[],;:_~=+/\\\"\'");
	int n = wcslen((wchar_t*)chareset);
	chareset[n] = WCHAR(0x7F);
	chareset[n + 1] = '\0';

	int nRc = CountRepeatedCharacters(pText, chareset);
	if(nRc >= 4){
		return false;
	}
	return true;
}

WCHAR *Dictionary::FilterText(const WCHAR *pText, int nValidToNotRatio /* *256 */, int &nQuality) {
	int nW, nNoW, nSW = 0, nSNoW = 0;
	int nNewLines = CountNewLines(pText);
	UINT nLen = (UINT)wcslen((wchar_t*)pText) + nNewLines * 2 + 1;
	WCHAR *pNewText = new WCHAR[nLen];
	WCHAR *pNT = pNewText;
	while(*pText != '\0'){
		const WCHAR *pEndl = CountValidWords(pText, nW, nNoW);
		nSW += nW;
		nSNoW += nNoW;
		if((nW << 8) >= nValidToNotRatio * nNoW){
			int nLn = int(pEndl - pText);
			memcpy(pNT, pText, nLn);
			pNT += nLn;
			*pNT++ = '\r';
			*pNT++ = '\n';
		}
		pText = pEndl;
		while(*pText == '\r' || *pText == '\n') ++pText;
	}
	*pNT = '\0';
	nQuality = ((nSNoW + nSW) < 1) ? 0 : ((nSW - nSNoW) << 8) / (nSNoW + nSW);
	return pNewText;
}
	
int Dictionary::InDictionary(const WCHAR *pLine, int pos) {
	if(!ISALETTER(pLine[pos])) return 0;
	int i=pos;
	while(i >= 0 && ISALETTER(pLine[i])){
		--i;
	}
	++i;
	if(LookUp(pLine + i)){
		return int(wcslen((wchar_t*)m_pCurrWord)) - pos + i;
	}
	return 0;
}

WCHAR *Dictionary::MergeLines(const WCHAR *pLine0, const WCHAR *pLine1, WCHAR *pRes) {
	int nWord0, nNoWord0, nWord1, nNoWord1;
	CountValidWords(pLine0, nWord0, nNoWord0);
	CountValidWords(pLine1, nWord1, nNoWord1);
	int nBestLine = 0;
	if(nWord1 * nNoWord0 > nWord0 * nNoWord1){
		nBestLine = 1;
	}
	int ln = int(wcslen((wchar_t*)pLine0)), n;
	if(!pRes){
		pRes = new WCHAR[ln];
	}
    int i;
	for(i=0; i<ln; ++i){
		if(pLine0[i] == pLine1[i]){
			pRes[i] = pLine0[i];
		}
		else if(pLine0[i] != ' ' && pLine1[i] == ' '){
			pRes[i] = pLine0[i];
		}
		else if(pLine0[i] == ' ' && pLine1[i] != ' '){
			pRes[i] = pLine1[i];
		}
		else if(n = InDictionary(pLine0, i)){
			while(n > 0){
				pRes[i] = pLine0[i];
				++i;
				--n;
			}
			--i;
		}
		else if(n = InDictionary(pLine1, i)){
			while(n > 0){
				pRes[i] = pLine1[i];
				++i;
				--n;
			}
			--i;
		}
		else if(ISALETTER(pLine0[i]) && !ISALETTER(pLine1[i])){
			pRes[i] = pLine0[i];
		}
		else if(ISALETTER(pLine1[i]) && !ISALETTER(pLine0[i])){
			pRes[i] = pLine1[i];
		}
		else{
			switch(nBestLine){
			case 0:
				pRes[i] = pLine0[i];
				break;
			case 1:
				pRes[i] = pLine1[i];
				break;
			}
		}
	}
	pRes[i] = '\0';
	return pRes;
}

const WCHAR *Dictionary::MergeText(WCHAR *pText0, WCHAR *pText1) {
	return m_textMerger.Merge(pText0, pText1, this);

}

WCHAR *Dictionary::OptimalMerge(WCHAR **ppTexts,  int nTexts, int nThreshold, BOOL *pNotOverlap) {
	return m_textMerger.OptimalMerge(ppTexts, nTexts, nThreshold, this, pNotOverlap);
}

void Dictionary::FilterText(WCHAR *pText, int nValidToNotRatio) {
	WCHAR *pNewText = new WCHAR[wcslen((wchar_t*)pText) + 1];
	WCHAR *pN = pNewText, *pT = pText;
	for(; *pT != '\0'; ++pT){
		if(*pT == '~'){
			*pN++ = ' ';
		}
		else if(ValidCharacter(*pT)){
			*pN++ = *pT;
		}
	}
	*pN = '\0';
	int nQual;
	WCHAR *pNT = FilterText(pNewText, nValidToNotRatio, nQual);
	wcscpy((wchar_t*)pText, (wchar_t*)pNT);
	delete [] pNT;
	delete [] pNewText;
}

BOOL Dictionary::FilterOutIllegalChars(const WCHAR *pSrc, WCHAR *pDest) const {

	BOOL bRet = FALSE;
	WCHAR *np = pDest;
	for(const WCHAR *p = pSrc; *p != '\0'; ++p){
		if(ValidCharacter(*p)){
			if(!bRet && *p != ' ' && *p != '\t' && *p != '\r' && *p != '\n'){
				bRet = TRUE;
			}
			*np++ = *p;
		}
	}
	*np = '\0';
	return bRet;
}

BOOL IsANumber(const wstring & sWord){
	int nLn =  sWord.length();
	int nDigitCount = 0;
	if(nLn < 2) return FALSE;
	for(int i = 0; i < nLn; ++i){
		if(ISDIGIT(sWord[i])) {
			nDigitCount++;
		}
	}
	return (nDigitCount * 2 >= int(sWord.length()));
}

Dictionary *Dictionary::SetLanguage(int nLangIndx) {
	CSingleLock sl(&g_ccDictionaryInit);
	sl.Lock();
	if(sl.IsLocked()) {
		if(m_nLangIndx == nLangIndx) {
			return mInstance;
		}
		delete mInstance;
		mInstance = NULL;
		m_nLangIndx = -1;
		switch(nLangIndx) {
	case RTK_ENGLISH:
		m_nLangIndx = nLangIndx;
		mInstance = new CEnglishDictionary;
		break;
		}
		if(mInstance) mInstance->Init(CHIPMONK_INSTALL_DIRECTORY);
		return mInstance;
	}
	return NULL;
}

Dictionary *g_pDict = NULL;

class CCompareWords{
public:
	bool operator () (const wstring & s1, const wstring & s2) {
		return g_pDict->CompareWords((WCHAR*)s1.c_str(), (WCHAR*)s2.c_str()) < 0;
	}
};


void Dictionary::Sort(const char *pSaveFileName) {
	g_pDict = this;
	set<wstring, CCompareWords> setWords;
	WCHAR pTmp[MAX_PATH];
	WCHAR *pCurrWord = NULL;
	for(pCurrWord = m_pDict; pCurrWord; pCurrWord = GetNextWord(pCurrWord)) {
		int nLength = WordLength(pCurrWord);
		memcpy(pTmp, pCurrWord, nLength * sizeof(WCHAR));
		pTmp[nLength] = 0;
		setWords.insert((wchar_t*)pTmp);
	}
	pCurrWord = m_pDict;
	for(set<wstring, CCompareWords>::const_iterator i = setWords.begin(); i != setWords.end(); ++i) {
		memcpy(pCurrWord, i->c_str(), i->length() * sizeof(WCHAR));
		pCurrWord[i->length()] = L'\r';
		pCurrWord[i->length()+1] = L'\n';
		pCurrWord += (i->length() + 2);
	}
	*pCurrWord = L'#';
	if(pSaveFileName) {
		FILE *fp = fopen(pSaveFileName, "wb");
		UCHAR pTmp[2];
		pTmp[0] = 0xff;
		pTmp[1] = 0xfe;
		fwrite(pTmp, 2, 1, fp);
		fwrite(m_pDict, (int(pCurrWord) - int(m_pDict)) / 2 - 1, 2, fp);
		fclose(fp);
	}
}

wstring & TrimString(wstring & s) {
	int nBeg = 0;
	for(; nBeg < int(s.length()) && s[nBeg] == L' '; ++nBeg);
	int nEnd = s.length() - 1;
	for(; nEnd >=nBeg && s[nEnd] == L' '; --nEnd);
	if(nEnd >=nBeg) {
			s = s.substr(nBeg, nEnd - nBeg + 1);
	}
	else {
		s = L"";
	}
	return s;
}

BOOL Dictionary::SpellCheck(wstring & sWord, BOOL speaking) {
	TrimString(sWord);
	return TRUE;
}
