/*************************************************************************** KDBSearchEngine.cpp - description ------------------- begin : Fri Sep 8 2000 copyright : (C) 2000 by Andrea Rizzi (C) 2005 by Stanislav Visnovsky email : rizzi@kde.org ***************************************************************************/ /* Translation search engine Copyright 2000 Andrea Rizzi rizzi@kde.org License GPL v 2.0 * * * In addition, as a special exception, the copyright holders give * * permission to link the code of this program with any edition of * * the TQt library by Trolltech AS, Norway (or with modified versions * * of TQt that use the same license as TQt), and distribute linked * * combinations including the two. You must obey the GNU General * * Public License in all respects for all of the code used other than * * TQt. If you modify this file, you may extend this exception to * * your version of the file, but you are not obligated to do so. If * * you do not wish to do so, delete this exception statement from * * your version. * */ #include #include #include #include #include #include #include #include #include #include #include #include "kapplication.h" #include "KDBSearchEngine.h" #include "dbscan.h" #include "errno.h" #include "stdio.h" #include "stdlib.h" #include #include "preferenceswidget.h" #include "dbse_factory.h" #include #include #include #include #include #include #include #include #include #include #include "database.h" #include "catalogsettings.h" #define BIGNUMBER 400000000 using namespace KBabel; KDBSearchEngine::KDBSearchEngine (TQObject * tqparent, const char *name): SearchEngine (tqparent, name) { edited = "unknown"; dm = 0; //Database Manager pw = 0; //Preference widget lang = ""; dbOpened = false; dbname = ""; lasterror = i18n ("No error"); connect (this, TQT_SIGNAL (hasError (const TQString &)), TQT_SLOT (setLastError (const TQString &))); IAmReady = true; // I don't know if it is a good idea, no DB loaded!!! scanInProgress = false; searching = false; stopNow = false; norm = false; // Normalize white space = FALSE comm = true; // Remove Comments = TRUE } KDBSearchEngine::~KDBSearchEngine () { } bool KDBSearchEngine::loadDatabase (TQString database, bool noask = false) { bool ret = true; bool asked = false; if (noask) asked = true; if (dm != 0) { delete dm; dm = 0; } TQDir dir (database); if (!dir.exists ()) { if (asked || KMessageBox::questionYesNo (0, i18n ("Database folder does not exist:\n%1\n" "Do you want to create it now?"). arg (database), TQString(), i18n("Create Folder"), i18n("Do Not Create")) == KMessageBox::Yes) { asked = true; TQStringList dirList; while (!dir.exists () && !dir.dirName ().isEmpty ()) { dirList.prepend (dir.dirName ()); dir.setPath (dir.path () + "/.."); } for (TQStringList::Iterator it = dirList.begin (); it != dirList.end (); ++it) { if (!dir.mkdir (*it)) { KMessageBox::sorry (0, i18n ("It was not possible to create folder %1"). arg (dir.path () + "/" + (*it))); ret = false; break; } dir.cd (*it); } } else { ret = false; } } if (ret) { // test, if there are both of ,old and standard databases TQString transFile = database + "/translations." + lang + ".db"; bool oldstuff = TQFile::exists (transFile + ",old"); bool newstuff = TQFile::exists (transFile); if (oldstuff && newstuff) { // there is an old db2 database, ask user if (KMessageBox:: questionYesNo (0, i18n ("

There are backup database files from previous versions " "of KBabel. However, another version of KBabel (probably from KDE 3.1.1 or 3.1.2) " "created a new database. As a result, your KBabel installation contains two versions " "of database files. Unfortunatelly, the old and new version " "can not be merged. You need to choose one of them.

" "If you choose the old version, the new one will be removed. " "If you choose the new version, the old database files will be left alone " "and you need to remove them manually. Otherwise this message will be displayed " "again (the old files are at $KDEHOME/share/apps/kbabeldict/dbsearchengine/*,old).

"), i18n ("Old Database Found"), i18n ("Use &Old Database"), i18n ("Use &New Database")) == KMessageBox::Yes) { // remove the new files TQFile::remove (transFile); TQFile::remove (database + "/wordsindex." + lang + ".db"); TQFile::remove (database + "/keysindex." + lang + ".db"); TQFile::remove (database + "/catalogsinfo." + lang + ".db"); // rename the old files KIO::NetAccess::copy (KURL (transFile + ",old"), KURL (transFile), 0); KIO::NetAccess:: copy (KURL (database + "/wordsindex." + lang + ".db,old"), KURL (database + "/wordsindex." + lang + ".db"), 0); KIO::NetAccess:: copy (KURL (database + "/keysindex." + lang + ".db,old"), KURL (database + "/keysindex." + lang + ".db"), 0); KIO::NetAccess:: copy (KURL (database + "/catalogsinfo." + lang + ".db,old"), KURL (database + "/catalogsinfo." + lang + ".db"), 0); TQFile::remove (transFile + ",old"); TQFile::remove (database + "/wordsindex." + lang + ".db,old"); TQFile::remove (database + "/keysindex." + lang + ".db,old"); TQFile::remove (database + "/catalogsinfo." + lang + ".db,old"); } } else if (oldstuff) { // rename the old files KIO::NetAccess::copy (KURL (transFile + ",old"), KURL (transFile), 0); KIO::NetAccess:: copy (KURL (database + "/wordsindex." + lang + ".db,old"), KURL (database + "/wordsindex." + lang + ".db"), 0); KIO::NetAccess:: copy (KURL (database + "/keysindex." + lang + ".db,old"), KURL (database + "/keysindex." + lang + ".db"), 0); KIO::NetAccess:: copy (KURL (database + "/catalogsinfo." + lang + ".db,old"), KURL (database + "/catalogsinfo." + lang + ".db"), 0); TQFile::remove (transFile + ",old"); TQFile::remove (database + "/wordsindex." + lang + ".db,old"); TQFile::remove (database + "/keysindex." + lang + ".db,old"); TQFile::remove (database + "/catalogsinfo." + lang + ".db,old"); } dm = new DataBaseManager (database, lang, this, "Database manager"); if (!dm->isOk ()) { if (asked || KMessageBox::questionYesNo (0, i18n ("Database files not found.\nDo you want to create them now?"), TQString(), i18n("Create"), i18n("Do Not Create")) == KMessageBox::Yes) { //fprintf(stderr,"SI\n"); ret = dm->createDataBase (database, lang); } else ret = false; } else ret = true; } //Wrong errore hangdling if (ret) totalRecord = dm->count (); else totalRecord = 0; return ret; } /* Set if the research have to consider multiple spaces as a single one. */ void KDBSearchEngine::setNormalizeSpace (bool normalize) { norm = normalize; } void KDBSearchEngine::setRemoveInternalComment (bool internalcomment) { comm = internalcomment; } /* Set if the research have to be Case Sensitive or not */ void KDBSearchEngine::setCaseSensitive (bool sensitive) { sens = sensitive; } /* Set the a string containing all char that must be ignored during the search. */ void KDBSearchEngine::setRemoveCharString (TQString chartoremove) { remchar = chartoremove; } /* Return true if there's a search in progress. */ bool KDBSearchEngine::isSearching () const { return searching; } /* Add a search string in the list of the string to search for. Returns the ID of the string in the list. Returns -1 if there is a problem (may be search in progress) */ int KDBSearchEngine::addSearchString (TQString searchString, int rule) { if (searching || scanInProgress) return -1; SearchEntry e; e.string = TQString (searchString); e.rules = rule; searchStringList.append (e); return searchStringList.count (); } /* Start the research in the database of all the string in the list */ bool KDBSearchEngine::startSearch (const TQString & str, uint pluralForm, const SearchFilter * filter) { if (autoUpdate) { updateSettings (); } int l1 = 0, l2 = 0; if (defSub1) l1 = defLimit1; if (defSub2) l2 = defLimit2; return startSingleSearch (str, l1, l2); } bool KDBSearchEngine::startSearchInTranslation (TQString s) { if (autoUpdate) { updateSettings (); } int l1 = 0, l2 = 0; if (defSub1) l1 = defLimit1; if (defSub2) l2 = defLimit2; return startSingleSearch (s, l1, l2, true); } bool KDBSearchEngine::openDb (bool noask = false) { if (!dbOpened) { dbOpened = loadDatabase (dbname, noask); //Try first to open it now if (!dbOpened) // Still not opened!! { hasError (i18n ("Cannot open the database")); return false; } } return true; } bool KDBSearchEngine::messagesForFilter (const SearchFilter * filter, TQValueList < SearchResult > &resultList, TQString & error) { int count = 0; stopNow = false; // Remove dirty. SearchResult m; if (searching) { error = i18n ("Another search has already been started"); return false; } if (scanInProgress) { error = i18n ("Unable to search now: a PO file scan is in progress"); return false; } if (!openDb ()) { error = i18n ("Unable to open the database"); return false; } if (totalRecord <= 0) { error = i18n ("Database empty"); return false; } TQString package = filter->location (); int step = (totalRecord / 30) + 1; int ntra, nref; int req = dm->searchCatalogInfo (package); if (req == -1) { error = i18n ("No entry for this package in the database."); return false; } DataBaseItem item; int i, h; kdDebug (0) << "looking for catalog " << req << endl; progressStarts (i18n ("Searching for %1 in database").arg (package)); for (item = dm->firstItem (); !item.isNull (); item = dm->nextItem ()) { count++; if (count % step == 0) { emit progress (100 * count / totalRecord); kapp->tqprocessEvents (100); } if (stopNow) { stopNow = false; searching = false; emit finished (); return true; // No error, stopped! } ntra = item.numTra; for (i = 0; i < ntra; i++) { nref = item.translations[i].numRef; for (h = 0; h < nref; h++) { if (item.translations[i].infoRef[h] == req) { m.found = item.key; m.translation = item.translations[i].translation; resultList.append (m); } } } } return true; } void KDBSearchEngine::repeat () { int count = 0; stopNow = false; // Remove dirty. if (searching) { return; } if (scanInProgress) { return; } if (!openDb ()) { return; } if (totalRecord <= 0) { return; } int step = (totalRecord / 30) + 1; int ntra, nref; DataBaseItem item; int i, h, tot; int req = dm->searchCatalogInfo ("kdelibs.po"); if (req == -1) kdDebug (0) << "No kdelibs.po found!" << endl; TQProgressDialog *pd = new TQProgressDialog (i18n ("Looking for repetitions"), i18n ("Stop"), 100); connect (this, TQT_SIGNAL (progress (int)), pd, TQT_SLOT (setProgress (int))); connect (this, TQT_SIGNAL (finished ()), pd, TQT_SLOT (close ())); connect (pd, TQT_SIGNAL (cancelled ()), this, TQT_SLOT (stopSearch ())); TQString txt = "// %1 repetitions, %2 translation(s)\ni18n(\"%3\");\n"; TQString id; int min; bool ok = false; min = TQInputDialog::getInteger (i18n ("Minimum Repetition"), i18n ("Insert the minimum number of repetitions for a string:"), 2, 1, 999999, 1, &ok); if (!ok) return; pd->show (); progressStarts (i18n ("Searching repeated string")); static TQTextEdit *mle = new TQTextEdit (); mle->clear (); bool inlibs; for (item = dm->firstItem (); !item.isNull (); item = dm->nextItem ()) { count++; if (count % step == 0) { emit progress (100 * count / totalRecord); kapp->tqprocessEvents (100); } if (stopNow) { stopNow = false; searching = false; emit finished (); return; // No error, stopped! } tot = 0; inlibs = false; ntra = item.numTra; for (i = 0; i < ntra; i++) { nref = item.translations[i].numRef; for (h = 0; h < nref; h++) if (item.translations[i].infoRef[h] == req) inlibs = true; tot += nref; } if (tot >= min && !inlibs) { id = item.key; id = id.tqreplace ("\n", "\"\n\""); mle->append (txt.arg (tot).arg (ntra).arg (id)); } } emit progress (100); emit finished (); mle->resize (400, 400); mle->show (); delete pd; return; } bool KDBSearchEngine::startSearchNow (int searchmode) { if (searchmode == -1) searchmode = mode; int count = 0; stopNow = false; // Remove dirty. clearResults (); if (searching) { hasError (i18n ("Another search has already been started")); return false; } if (scanInProgress) { hasError (i18n ("Unable to search now: a PO file scan is in progress")); return false; } if (!openDb ()) return false; if (totalRecord <= 0) { hasError (i18n ("Database empty")); return false; } searching = true; emit started (); bool allkey = (searchmode == MD_ALL_GOOD_KEYS); bool equal, tqcontains, contained, regexp, intra; intra = searchmode & MD_IN_TRANSLATION; TQString msgIdFound; TQString msgId; TQString msgStr; //TQString msgIdRequested; SearchResult *aresult; TranslationInfo *adescription; SearchList searchList; int i, len, files, h; len = remchar.length (); int n, m; //,word; TQString *id; TQString mainRequest = searchStringList[0].string; SearchList::Iterator it, it1; TQString *idMod; bool foundSomething = false; searchList = searchStringList; //Create a copy and modify it if (!allkey) { for (it = searchList.begin (); it != searchList.end (); ++it) { idMod = &((*it).string); int pos; for (i = 0; i < len; i++) { while ((pos = idMod->tqfind (remchar.at (i))) != -1) idMod->remove (pos, 1); } if (comm) idMod->tqreplace (TQRegExp ("\\_\\:.*\\\\n"), ""); //Read it from catalog !!! (NOT ONLY HERE) if (norm) idMod->simplifyWhiteSpace (); if (!sens) *idMod = idMod->upper (); } } timeval now; gettimeofday (&now, NULL); //fprintf(stderr,"\n%ld.%ld\n",now.tv_sec,now.tv_usec); //double tim=1.0*now.tv_usec/1000000.0+now.tv_sec; int pos; DataBaseItem item; //Now we can browse the whole db or the "good keys" TQValueList < KeyAndScore > goodkeys; int totalprogress; bool gk = (searchmode == MD_GOOD_KEYS) || allkey; int k = 0; if (gk) { goodkeys = searchWords (mainRequest, thre, threorig, listmax); //FIX IT, mainReq? if (stopNow) { stopNow = false; searching = false; emit finished (); return false; } if (goodkeys.count () == 0) gk = false; // if no good keys, use the whole database } // prepare progress values totalprogress = gk ? goodkeys.count () : totalRecord; int step = (totalprogress / 30) + 1; if( step > 100 ) step = 100; emit progress (0); kapp->tqprocessEvents (100); if (stopNow) { stopNow = false; searching = false; emit finished (); return true; // No error, stopped! } for (item = gk ? (dm->getItem (goodkeys[0])) : (dm->firstItem ()); !item.isNull (); item = gk ? (dm->getItem (goodkeys[++k])) : (dm->nextItem ())) { // Emit progress, process event and check stop now if (count % step == 0) { emit progress (100 * count / /*TQMAX( */ totalprogress /*,1) */ ); kapp->tqprocessEvents (100); if (stopNow) { stopNow = false; searching = false; emit finished (); return true; // No error, stopped! } } // fprintf(stderr,"%s\n",(const char *)item.key.utf8()); msgIdFound = item.key; //Check if this is OK with UTF8 // searchmode && MD_IN_TRANSLATION) count++; msgId = msgIdFound; if (!allkey) { //Remove character in list of character to be ignored for (i = 0; i < len; i++) while ((pos = msgId.tqfind (remchar.at (i))) != -1) msgId.remove (pos, 1); //Remove context information from id found if (comm) msgId.tqreplace (TQRegExp ("\\_\\:.*\\\\n"), ""); if (norm) msgId.simplifyWhiteSpace (); if (!sens) msgId = msgId.upper (); } it = searchList.begin (); idMod = &((*it).string); bool foundExact = false; for (it1 = searchStringList.begin (); it1 != searchStringList.end (); it1++) { id = &((*it1).string); uint nn = 0; do { if (intra) { msgId = item.translations[nn].translation; if (!allkey) { //Remove character in list of character to be ignored for (i = 0; i < len; i++) while ((pos = msgId.tqfind (remchar.at (i))) != -1) msgId.remove (pos, 1); //Remove context information from id found if (comm) msgId. tqreplace (TQRegExp ("\\_\\:.*\\\\n"), ""); if (norm) msgId.simplifyWhiteSpace (); if (!sens) msgId = msgId.upper (); } } int rules = (*it).rules; if (rules & Equal) equal = (*idMod == msgId); else equal = false; if (rules & Contains) tqcontains = idMod->tqcontains (msgId); else tqcontains = false; if (rules & Contained) contained = msgId.tqcontains (*idMod); else contained = false; if (!foundExact && (rules & RegExp)) { TQRegExp reg (*idMod); regexp = (reg.search (msgId) != -1); } else regexp = false; nn++; } while (intra && nn < item.numTra); if (equal || tqcontains || contained || regexp || allkey) { if (equal) foundExact = true; m = item.numTra; //Translations found. for (n = 0; n < m; n++) { foundSomething = true; msgStr = item.translations[n].translation; files = item.translations[n].numRef; aresult = new SearchResult (); results.setAutoDelete (true); if (!gk) aresult->score = score (mainRequest, msgIdFound); else aresult->score = goodkeys[k].score; if (intra) aresult->score = score (mainRequest, item.translations[n].translation); SearchResult *s = 0; for (s = results.first (); s != 0; s = results.next ()) { if (s->score > aresult->score) { results.insert (results.at (), aresult); break; } } if (s == 0) //no break or empty list results.append (aresult); /* if(*id==msgIdFound) //Put it first of the list results.prepend(aresult); else results.append(aresult); */ aresult->requested = *id; aresult->found = msgIdFound; aresult->translation = msgStr; aresult->descriptions.setAutoDelete (true); for (h = 0; h < files; h++) { aresult->descriptions.append (adescription = new TranslationInfo ()); int rr = item.translations[n].infoRef[h]; InfoItem info = dm->getCatalogInfo (rr); adescription->location = info.catalogName; adescription->translator = info.lastTranslator; adescription->filePath = info.lastFullPath; } emit numberOfResultsChanged (results.count ()); emit resultFound (aresult); // if(*id==msgIdFound) //Put it first of the list so th order change emit resultsReordered (); } } // idMod=searchList.next(); it++; idMod = &((*it).string); } } gettimeofday (&now, NULL); //fprintf(stderr,"%ld.%ld\n",now.tv_sec,now.tv_usec); //fprintf(stderr,"Finish, %d (of %d) records in %f seconds!!\n",count,totalRecord, 1.0*now.tv_usec/1000000.0+now.tv_sec-tim); emit progress (100); emit finished (); searching = false; return true; //foundSomething; } /* Start a search for a single string */ bool KDBSearchEngine::startSingleSearch (TQString searchString, unsigned int pattern1Limit, unsigned int /*pattern2Limit */ , bool inTranslation) { /* Check Ret value. */ unsigned int nw = 0; int in = 0, len = 0; clearList (); addSearchString (searchString, defRule); TQRegExp reg ("[a-zA-Z0-9_%" /*+remchar */ + regaddchar + "]+"); while ((in = reg.search (searchString, in + len)) != -1) { nw++; len = reg.matchedLength (); } in = 0; len = 0; // fprintf(stderr,"asas %d\n",nw); if (mode == MD_ALL_GOOD_KEYS && !inTranslation) return startSearchNow (); if ((nw < pattern1Limit) && (nw > 1)) for (unsigned int k = 0; k < nw; k++) { in = reg.search (searchString, in + len); len = reg.matchedLength (); TQString regToAdd = searchString; regToAdd.tqreplace (in, len, "[a-zA-Z0-9_%" + regaddchar + "]*"); regToAdd.append ("$"); regToAdd.prepend ("^"); // fprintf(stderr,"%s",(const char *)regToAdd.local8Bit()); addSearchString (regToAdd, RegExp); } if (inTranslation) return startSearchNow (MD_IN_TRANSLATION); else return startSearchNow (); return false; } /* Start a search for a list of string */ //bool KDBSearchEngine::startListSearch(TQPtrList searchStrList) //{ // searchStringList=searchStrList; // return startSearchNow(); //} /* Stop the current search */ void KDBSearchEngine::setLanguageCode (const TQString & ll) { if (ll == lang) return; lang = ll; if (dbOpened) //if opened open it again with new code, what about close before open ? dbOpened = loadDatabase (dbname); } void KDBSearchEngine::setLanguage (const TQString & languageCode, const TQString & /*languageName */ ) { setLanguageCode (languageCode); } void KDBSearchEngine::stopSearch () { stopNow = true; } void KDBSearchEngine::clearList () { searchStringList.clear (); } bool KDBSearchEngine::isReady () const { return IAmReady; } void KDBSearchEngine::saveSettings (KConfigBase * config) { // updateSettings(); //maybe with autoupdate KConfigGroupSaver cgs (config, "KDBSearchEngine"); #if KDE_IS_VERSION(3,1,3) config->writePathEntry ("Filename", dbname); #else config->writeEntry ("Filename", dbname); #endif config->writeEntry ("Language", lang); config->writeEntry ("CaseSensitive", sens); config->writeEntry ("Normalize", norm); config->writeEntry ("RemoveContext", comm); config->writeEntry ("Rules", defRule); config->writeEntry ("Limit1", defLimit1); config->writeEntry ("Limit2", defLimit2); config->writeEntry ("Substitution1", defSub1); config->writeEntry ("Substitution2", defSub2); config->writeEntry ("RegExp", regaddchar); config->writeEntry ("RemoveCharacter", remchar); config->writeEntry ("Threshold1", thre); config->writeEntry ("Threshold2", threorig); config->writeEntry ("ListMax", listmax); config->writeEntry ("Mode", mode); config->writeEntry ("CommonThrs", commonthre); config->writeEntry ("ReturnNothing", retnot); config->writeEntry ("AutoAuthor", autoauthor); config->writeEntry ("AutoUp", autoup); } void KDBSearchEngine::readSettings (KConfigBase * config) { TQString newName; KConfigGroupSaver cgs (config, "KDBSearchEngine"); TQString defaultLang; TQString oldLang = lang; Defaults::Identity def; defaultLang = def.languageCode (); lang = config->readEntry ("Language", defaultLang); TQString defaultDir; KStandardDirs *dirs = KGlobal::dirs (); if (dirs) { defaultDir = dirs->saveLocation ("data"); if (defaultDir.right (1) != "/") defaultDir += "/"; defaultDir += "kbabeldict/dbsearchengine"; } newName = config->readPathEntry ("Filename", defaultDir); if (newName != dbname || oldLang != lang) { dbname = newName; if (dbOpened) //Reload only if it is opened dbOpened = loadDatabase (dbname); } sens = config->readBoolEntry ("CaseSensitive", false); norm = config->readBoolEntry ("Normalize", true); comm = config->readBoolEntry ("RemoveContext", true); defRule = config->readNumEntry ("Rules", 1); defLimit1 = config->readNumEntry ("Limit1", 20); defLimit2 = config->readNumEntry ("Limit2", 8); thre = config->readNumEntry ("Threshold1", 50); threorig = config->readNumEntry ("Threshold2", 50); listmax = config->readNumEntry ("ListMax", 500); mode = config->readNumEntry ("Mode", MD_GOOD_KEYS); defSub1 = config->readBoolEntry ("Substitution1", true); defSub2 = config->readBoolEntry ("Substitution2", false); regaddchar = config->readEntry ("RegExp"); remchar = config->readEntry ("RemoveCharacter", "&.:"); commonthre = config->readNumEntry ("CommonThrs", 300); retnot = config->readBoolEntry ("ReturnNothing", false); autoauthor = config->readEntry ("AutoAuthor"); autoup = config->readBoolEntry ("AutoUp", true); setSettings (); } PrefWidget * KDBSearchEngine::preferencesWidget (TQWidget * tqparent) { pw = new PreferencesWidget (tqparent); setSettings (); connect (pw, TQT_SIGNAL (restoreNow ()), this, TQT_SLOT (setSettings ())); connect (pw, TQT_SIGNAL (applyNow ()), this, TQT_SLOT (updateSettings ())); connect (pw, TQT_SIGNAL (destroyed ()), this, TQT_SLOT (prefDestr ())); connect (pw->dbpw->scanPB_2, TQT_SIGNAL (clicked ()), this, TQT_SLOT (scan ())); connect (pw->dbpw->scanrecPB, TQT_SIGNAL (clicked ()), this, TQT_SLOT (scanRecur ())); connect (pw->dbpw->scanFilePB, TQT_SIGNAL (clicked ()), this, TQT_SLOT (scanFile ())); connect (pw->dbpw->repeatPB, TQT_SIGNAL (clicked ()), this, TQT_SLOT (repeat ())); return pw; } void KDBSearchEngine::scanRecur () { if (scanInProgress) return; updateSettings (); if (!openDb ()) return; scanInProgress = true; PoScanner *sca = new PoScanner (dm, this, "Po Scanner"); TQString cvsdir; cvsdir = KFileDialog::getExistingDirectory ("", 0, i18n ("Select Folder to Scan Recursively")); if (cvsdir.isEmpty ()) { scanInProgress = false; return; } if (pw) { connect (sca, TQT_SIGNAL (patternProgress (int)), pw->dbpw->totalPB, TQT_SLOT (setProgress (int))); connect (sca, TQT_SIGNAL (fileLoading (int)), pw->dbpw->loadingPB, TQT_SLOT (setProgress (int))); connect (sca, TQT_SIGNAL (fileProgress (int)), pw->dbpw->processPB, TQT_SLOT (setProgress (int))); } connect (sca, TQT_SIGNAL (patternProgress (int)), TQT_SIGNAL (progress (int))); //Kbabel progress bar connect (sca, TQT_SIGNAL (added (int)), pw, TQT_SLOT (setEntries (int))); connect (sca, TQT_SIGNAL (filename (TQString)), pw, TQT_SLOT (setName (TQString))); progressStarts (i18n ("Scanning folder %1").arg (cvsdir)); connect (sca, TQT_SIGNAL (patternFinished ()), TQT_SIGNAL (progressEnds ())); sca->scanPattern (cvsdir, "*.po", true); disconnect (this, TQT_SIGNAL (progress (int))); //disconnect(TQT_SIGNAL(patternStarted()),this,TQT_SIGNAL(started()) ); disconnect (this, TQT_SIGNAL (progressEnds ())); if (pw) { disconnect (pw->dbpw->totalPB, TQT_SLOT (setProgress (int))); disconnect (pw->dbpw->loadingPB, TQT_SLOT (setProgress (int))); disconnect (pw->dbpw->processPB, TQT_SLOT (setProgress (int))); } totalRecord = dm->count (); scanInProgress = false; dm->sync (); delete sca; } void KDBSearchEngine::scan () { if (scanInProgress) return; updateSettings (); if (!openDb ()) return; scanInProgress = true; PoScanner *sca = new PoScanner (dm, this, "Po Scanner"); TQString cvsdir; cvsdir = KFileDialog::getExistingDirectory ("", 0, i18n ("Select Folder to Scan")); if (cvsdir.isEmpty ()) { scanInProgress = false; return; } if (pw) { connect (sca, TQT_SIGNAL (patternProgress (int)), pw->dbpw->totalPB, TQT_SLOT (setProgress (int))); connect (sca, TQT_SIGNAL (fileLoading (int)), pw->dbpw->loadingPB, TQT_SLOT (setProgress (int))); connect (sca, TQT_SIGNAL (fileProgress (int)), pw->dbpw->processPB, TQT_SLOT (setProgress (int))); } connect (sca, TQT_SIGNAL (patternProgress (int)), TQT_SIGNAL (progress (int))); progressStarts (i18n ("Scanning folder %1").arg (cvsdir)); connect (sca, TQT_SIGNAL (patternFinished ()), TQT_SIGNAL (progressEnds ())); connect (sca, TQT_SIGNAL (added (int)), pw, TQT_SLOT (setEntries (int))); connect (sca, TQT_SIGNAL (filename (TQString)), pw, TQT_SLOT (setName (TQString))); sca->scanPattern (cvsdir, "*.po", false); disconnect (this, TQT_SIGNAL (progress (int))); //disconnect(TQT_SIGNAL(patternStarted()),this,TQT_SIGNAL(started()) ); disconnect (this, TQT_SIGNAL (progressEnds ())); if (pw) { disconnect (pw->dbpw->totalPB, TQT_SLOT (setProgress (int))); disconnect (pw->dbpw->loadingPB, TQT_SLOT (setProgress (int))); disconnect (pw->dbpw->processPB, TQT_SLOT (setProgress (int))); } totalRecord = dm->count (); scanInProgress = false; dm->sync (); delete sca; } void KDBSearchEngine::scanFile () { if (scanInProgress) return; updateSettings (); if (!openDb ()) return; scanInProgress = true; PoScanner *sca = new PoScanner (dm, this, "Po Scanner"); TQString cvsdir; pw->dbpw->totalPB->setProgress (0); cvsdir = KFileDialog::getOpenFileName ("", "*.po", 0, i18n ("Select PO File to Scan")); if (cvsdir.isEmpty ()) { scanInProgress = false; return; } if (pw) { connect (sca, TQT_SIGNAL (fileLoading (int)), pw->dbpw->loadingPB, TQT_SLOT (setProgress (int))); connect (sca, TQT_SIGNAL (fileProgress (int)), pw->dbpw->processPB, TQT_SLOT (setProgress (int))); } connect (sca, TQT_SIGNAL (fileProgress (int)), TQT_SIGNAL (progress (int))); progressStarts (i18n ("Scanning file %1").arg (directory (cvsdir, 0))); connect (sca, TQT_SIGNAL (fileFinished ()), TQT_SIGNAL (progressEnds ())); connect (sca, TQT_SIGNAL (added (int)), pw, TQT_SLOT (setEntries (int))); connect (sca, TQT_SIGNAL (filename (TQString)), pw, TQT_SLOT (setName (TQString))); sca->scanFile (cvsdir); sca->disconnect (TQT_SIGNAL (fileProgress (int)), this, TQT_SIGNAL (progress (int))); //disconnect(TQT_SIGNAL(patternStarted()),this,TQT_SIGNAL(started()) ); sca->disconnect (TQT_SIGNAL (fileFinished ()), this, TQT_SIGNAL (progressEnds ())); if (pw) { disconnect (pw->dbpw->loadingPB, TQT_SLOT (setProgress (int))); disconnect (pw->dbpw->processPB, TQT_SLOT (setProgress (int))); } totalRecord = dm->count (); scanInProgress = false; dm->sync (); delete sca; } const KAboutData * KDBSearchEngine::about () const { return DbSeFactory::instance ()->aboutData (); } TQString KDBSearchEngine::name () const { return i18n ("Translation Database"); } TQString KDBSearchEngine::id () const { return TQString ("KDBSearchEngine"); } TQString KDBSearchEngine::lastError () { return lasterror; } void KDBSearchEngine::prefDestr () { pw = 0; } void KDBSearchEngine::setSettings () { if (pw == 0) return; pw->dbpw->dirInput->setURL (dbname); pw->dbpw->caseSensitiveCB->setChecked (sens); pw->dbpw->normalizeCB->setChecked (norm); pw->dbpw->removeContextCB->setChecked (comm); pw->dbpw->oneWordSubCB->setChecked (defSub1); pw->dbpw->twoWordSubCB->setChecked (defSub2); if (defRule == 8) pw->dbpw->RegExpRB->setChecked (true); else { pw->dbpw->normalTextRB->setChecked (true); pw->dbpw->equalCB->setChecked (defRule & Equal); pw->dbpw->containsCB->setChecked (defRule & Contains); pw->dbpw->containedCB->setChecked (defRule & Contained); } pw->dbpw->oneWordSubSB->setValue (defLimit1); pw->dbpw->twoWordSubSB->setValue (defLimit2); pw->dbpw->maxSB->setValue (listmax); pw->dbpw->thresholdSL->setValue (thre); pw->dbpw->thresholdOrigSL->setValue (threorig); pw->dbpw->allRB->setChecked (mode == MD_ALL_DB); pw->dbpw->slistRB->setChecked (mode == MD_GOOD_KEYS); pw->dbpw->rlistRB->setChecked (mode == MD_ALL_GOOD_KEYS); pw->dbpw->nothingCB->setChecked (retnot); pw->dbpw->freqSB->setValue (commonthre); pw->dbpw->regExpLE->setText (regaddchar); pw->dbpw->ignoreLE->setText (remchar); pw->dbpw->authorLE->setText (autoauthor); pw->dbpw->autoAddCB_2->setChecked (autoup); } void KDBSearchEngine::updateSettings () { if (pw == 0) return; TQString newName = pw->dbpw->dirInput->url (); if (newName != dbname) { kdDebug (0) << "Database changed" << endl; dbname = newName; if (dbOpened) dbOpened = loadDatabase (dbname); } sens = pw->dbpw->caseSensitiveCB->isChecked (); norm = pw->dbpw->normalizeCB->isChecked (); comm = pw->dbpw->removeContextCB->isChecked (); int tmpRule = 0; if (pw->dbpw->RegExpRB->isChecked ()) tmpRule = RegExp; else { if (pw->dbpw->equalCB->isChecked ()) tmpRule += Equal; if (pw->dbpw->containsCB->isChecked ()) tmpRule += Contains; if (pw->dbpw->containedCB->isChecked ()) tmpRule += Contained; } defRule = tmpRule; defLimit1 = pw->dbpw->oneWordSubSB->text ().toInt (); defLimit2 = pw->dbpw->twoWordSubSB->text ().toInt (); defSub1 = pw->dbpw->oneWordSubCB->isChecked (); defSub2 = pw->dbpw->twoWordSubCB->isChecked (); listmax = pw->dbpw->maxSB->value (); thre = pw->dbpw->thresholdSL->value (); threorig = pw->dbpw->thresholdOrigSL->value (); if (pw->dbpw->allRB->isChecked ()) mode = MD_ALL_DB; if (pw->dbpw->slistRB->isChecked ()) mode = MD_GOOD_KEYS; if (pw->dbpw->rlistRB->isChecked ()) mode = MD_ALL_GOOD_KEYS; regaddchar = pw->dbpw->regExpLE->text (); remchar = pw->dbpw->ignoreLE->text (); retnot = pw->dbpw->nothingCB->isChecked (); commonthre = pw->dbpw->freqSB->value (); autoauthor = pw->dbpw->authorLE->text (); autoup = pw->dbpw->autoAddCB_2->isChecked (); } void KDBSearchEngine::setLastError (const TQString & er) { lasterror = er; } TQString KDBSearchEngine::translate (const TQString & text, const uint pluralForm) { if (!openDb ()) return TQString(); /* if(!dbOpened) { dbOpened=loadDatabase(dbname); //Try first to open it now if(!dbOpened) // Still not opened!! { //emit anerror hasError(i18n("Database not opened")); return TQString(); } } */ DataBaseItem dbit = dm->getItem (text); if (dbit.isNull ()) return TQString(); if (dbit.numTra == 1) return dbit.translations[0].translation; uint32 n = dbit.numTra; uint32 max = 0, nmax = 0; for (uint32 i = 0; i < n; i++) if (dbit.translations[i].numRef > max) { nmax = i; max = dbit.translations[i].numRef; } return dbit.translations[nmax].translation; } TQValueList < KeyAndScore > KDBSearchEngine::searchWords (TQString phrase, int threshold, int thresholdorig, uint32 max) { TQValueList < TQString > wordlist; if (!openDb ()) { TQValueList < KeyAndScore > a; return a; } progressStarts (i18n ("Searching words")); TQValueList < TQString >::Iterator wlit; wordlist = dm->wordsIn (phrase); int nw = wordlist.count (); //TQMemArray wi(nw); TQMemArray < uint32 > numofloc (nw), currentloc (nw); TQMemArray < int > score (nw); TQMemArray < uint32 * >loc (nw), locorig (nw); TQValueList < uint32 > resloc; TQValueList < int > resfound; TQValueList < KeyAndScore > keylist; //wi.resize(wordlist.count()); int totalprogress = 0; int totrec = dm->count (); uint32 cthre = totrec * commonthre / 10000; int i = 0, common = 0; for (wlit = wordlist.begin (); wlit != wordlist.end (); ++wlit) { WordItem wi = dm->getWordLocations (*wlit); if (!wi.notFound ()) { if (wi.count < cthre) score[i] = 1; else { score[i] = 0; common++; } locorig[i] = loc[i] = wi.locations; totalprogress += numofloc[i] = wi.count; currentloc[i] = 0; // score[i]=wi.score; //wi[i]=WordItem(wi[i]); //wi[i].locations.detach(); i++; // } // else // common++; } } bool cs = (common == nw); //All words are common; if (totalprogress == 0) totalprogress = 1; int step = totalprogress / 30 + 1; int count = 0; int thrs = (wordlist.count () * threshold) / 100; if (thrs < 1) thrs = 1; // whole database ??? int tot = i; //nt32 jmin=0; int found; uint32 min; //Big ? bool empty = false; while (!empty) { empty = true; found = retnot ? common : 0; if (thrs <= found) thrs = found + 1; // whole database ??? min = BIGNUMBER; for (int j = 0; j < tot; j++) if (cs || score[j]) { if (numofloc[j] > currentloc[j]) // Check if there's still something to do. empty = false; if (loc[j][0] < min) //Found the minimum head min = loc[j][0]; } if (min != BIGNUMBER) { for (int j = 0; j < tot; j++) if (cs || score[j]) { if (loc[j][0] == min) //Count the heads, move forward { found++; count++; //check stopnow here if (count % step == 0) { emit progress (100 * count / totalprogress); kapp->tqprocessEvents (100); } if (stopNow) { return keylist; } currentloc[j]++; if (numofloc[j] == currentloc[j]) //End reached loc[j][0] = BIGNUMBER; //so set head to a big number else //Go on.. { loc[j]++; } } } //end of for bool inserted = false; if (found >= thrs) { //Words count in key. int nword = 0; int in = 0, len = 0; TQString keyst = dm->getKey (min); TQRegExp reg ("[a-zA-Z0-9_%" /*+remchar */ + regaddchar + "]+"); while ((in = reg.search (keyst, in + len)) != -1) { nword++; len = reg.matchedLength (); } if (found >= nword * thresholdorig / 100) // { if (resfound.count () <= max || (*resfound.end () < found)) if ((*resfound.end ()) >= found) { inserted = true; resloc.append (min); resfound.append (found); } else for (uint32 j = 0; j < resloc.count (); j++) { if (resfound[j] < found || (resfound[j] == found && 0)) //Orig word { resloc.insert (resloc.at (j), min); resfound.insert (resfound. at (j), found); inserted = true; break; } } if (!inserted) { resloc.append (min); resfound.append (found); } } } } } int nres = (resloc.count () < max) ? resloc.count () : max; for (int j = 0; j < nres; j++) { TQString strkey = dm->getKey (resloc[j]); int stdscore = KDBSearchEngine::score (phrase, strkey); int sc = 0; if (stdscore < 99) { int in = 0, len = 0, nword = 0; int remove = retnot ? common : 0; TQRegExp reg ("[a-zA-Z0-9_%" /*+remchar */ + regaddchar + "]+"); while ((in = reg.search (strkey, in + len)) != -1) { nword++; len = reg.matchedLength (); } // kdDebug(0) << nword << "NWORD " << resfound[j] << "FOUND " // << resfound[j]-remove << "REAL " << remove << "to be remove " << endl; if (nword < 1) nword = 1; sc = 70 - resfound[j] * 70 / nw + abs (nword - (resfound[j] - remove)) * 30 / nword + 2; sc = 100 - sc; // kdDebug(0) <<" Score : " << sc << endl; } else sc = stdscore; KeyAndScore key (strkey, sc); // kdDebug(0) << (TQString) key << " [" << key.score << "]" << endl; keylist.append (key); } //kdDebug(0) << "Here!" << endl; for (int j = 0; j < tot; j++) { free (locorig[j]); } progressStarts (i18n ("Process output")); return keylist; } void KDBSearchEngine::stringChanged (const TQStringList & o, const TQString & translated, const uint, const TQString &) { TQString orig = o.first (); // FIXME: plural forms // skip empty originals or translated texts if (orig.isEmpty () || translated.isEmpty ()) return; if (autoup) { if (openDb (true)) //true= no ask { dm->putNewTranslation (orig, translated, dm->catalogRef (directory (edited, 0), autoauthor, edited)); //kdDebug(0) << "Changed " << orig << " " << translated << endl; dm->sync (); } } } void KDBSearchEngine::setEditedFile (const TQString & file) { edited = file; //kdDebug(0) << edited << endl; } KeyAndScore::KeyAndScore (const TQString & a, int sc): TQString (a) { score = sc; } KeyAndScore::KeyAndScore ():TQString () { score = 0; } #include "KDBSearchEngine.moc"