diff options
Diffstat (limited to 'experimental/tqtinterface/qt4/tools/linguist/linguist/simtexth.cpp')
| -rw-r--r-- | experimental/tqtinterface/qt4/tools/linguist/linguist/simtexth.cpp | 239 |
1 files changed, 239 insertions, 0 deletions
diff --git a/experimental/tqtinterface/qt4/tools/linguist/linguist/simtexth.cpp b/experimental/tqtinterface/qt4/tools/linguist/linguist/simtexth.cpp new file mode 100644 index 000000000..2d139651e --- /dev/null +++ b/experimental/tqtinterface/qt4/tools/linguist/linguist/simtexth.cpp @@ -0,0 +1,239 @@ +/********************************************************************** +** Copyright (C) 2000-2008 Trolltech ASA. All rights reserved. +** +** This file is part of TQt Linguist. +** +** This file may be used under the terms of the GNU General +** Public License versions 2.0 or 3.0 as published by the Free +** Software Foundation and appearing in the files LICENSE.GPL2 +** and LICENSE.GPL3 included in the packaging of this file. +** Alternatively you may (at your option) use any later version +** of the GNU General Public License if such license has been +** publicly approved by Trolltech ASA (or its successors, if any) +** and the KDE Free TQt Foundation. +** +** Please review the following information to ensure GNU General +** Public Licensing requirements will be met: +** http://trolltech.com/products/qt/licenses/licensing/opensource/. +** If you are unsure which license is appropriate for your use, please +** review the following information: +** http://trolltech.com/products/qt/licenses/licensing/licensingoverview +** or contact the sales department at sales@trolltech.com. +** +** Licensees holding valid TQt Commercial licenses may use this file in +** accordance with the TQt Commercial License Agreement provided with +** the Software. +** +** This file is provided "AS IS" with NO WARRANTY OF ANY KIND, +** INCLUDING THE WARRANTIES OF DESIGN, MERCHANTABILITY AND FITNESS FOR +** A PARTICULAR PURPOSE. Trolltech reserves all rights not granted +** herein. +** +**********************************************************************/ + +#include "simtexth.h" + +#include <metatranslator.h> + +#include <tqcstring.h> +#include <tqdict.h> +#include <tqmap.h> +#include <tqstring.h> +#include <tqstringlist.h> +#include <tqvaluelist.h> + +#include <string.h> + +typedef TQValueList<MetaTranslatorMessage> TML; + +/* + How similar are two texts? The approach used here relies on co-occurrence + matrices and is very efficient. + + Let's see with an example: how similar are "here" and "hither"? The + co-occurrence matrix M for "here" is M[h,e] = 1, M[e,r] = 1, M[r,e] = 1, and 0 + elsewhere; the matrix N for "hither" is N[h,i] = 1, N[i,t] = 1, ..., + N[h,e] = 1, N[e,r] = 1, and 0 elsewhere. The union U of both matrices is the + matrix U[i,j] = max { M[i,j], N[i,j] }, and the intersection V is + V[i,j] = min { M[i,j], N[i,j] }. The score for a pair of texts is + + score = (sum of V[i,j] over all i, j) / (sum of U[i,j] over all i, j), + + a formula suggested by Arnt Gulbrandsen. Here we have + + score = 2 / 6, + + or one third. + + The implementation differs from this in a few details. Most importantly, + repetitions are ignored; for input "xxx", M[x,x] equals 1, not 2. +*/ + +/* + Every character is assigned to one of 20 buckets so that the co-occurrence + matrix requires only 20 * 20 = 400 bits, not 256 * 256 = 65536 bits or even + more if we want the whole Unicode. Which character falls in which bucket is + arbitrary. + + The second half of the table is a replica of the first half, because of + laziness. +*/ +static const int indexOf[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// ! " # $ % & ' ( ) * + , - . / + 0, 2, 6, 7, 10, 12, 15, 19, 2, 6, 7, 10, 12, 15, 19, 0, +// 0 1 2 3 4 5 6 7 8 9 : ; < = > ? + 1, 3, 4, 5, 8, 9, 11, 13, 14, 16, 2, 6, 7, 10, 12, 15, +// @ A B C D E F G H I J K L M N O + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14, +// P Q R S T U V W X Y Z [ \ ] ^ _ + 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0, +// ` a b c d e f g h i j k l m n o + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14, +// p q r s t u v w x y z { | } ~ + 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2, 6, 7, 10, 12, 15, 19, 2, 6, 7, 10, 12, 15, 19, 0, + 1, 3, 4, 5, 8, 9, 11, 13, 14, 16, 2, 6, 7, 10, 12, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14, + 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14, + 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0 +}; + +/* + The entry bitCount[i] (for i between 0 and 255) is the number of bits used to + represent i in binary. +*/ +static const int bitCount[256] = { + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8 +}; + +struct CoMatrix +{ + /* + The matrix has 20 * 20 = 400 entries. This requires 50 bytes, or 13 + words. Some operations are performed on words for more efficiency. + */ + union { + TQ_UINT8 b[52]; + TQ_UINT32 w[13]; + }; + + CoMatrix() { memset( b, 0, 52 ); } + CoMatrix( const char *text ) { + char c = '\0', d; + memset( b, 0, 52 ); + /* + The Knuth books are not in the office only for show; they help make + loops 30% faster and 20% as readable. + */ + while ( (d = *text) != '\0' ) { + setCoocc( c, d ); + if ( (c = *++text) != '\0' ) { + setCoocc( d, c ); + text++; + } + } + } + + void setCoocc( char c, char d ) { + int k = indexOf[(uchar) c] + 20 * indexOf[(uchar) d]; + b[k >> 3] |= k & 0x7; + } + + int worth() const { + int w = 0; + for ( int i = 0; i < 50; i++ ) + w += bitCount[b[i]]; + return w; + } +}; + +static inline CoMatrix reunion( const CoMatrix& m, const CoMatrix& n ) +{ + CoMatrix p; + for ( int i = 0; i < 13; i++ ) + p.w[i] = m.w[i] | n.w[i]; + return p; +} + +static inline CoMatrix intersection( const CoMatrix& m, const CoMatrix& n ) +{ + CoMatrix p; + for ( int i = 0; i < 13; i++ ) + p.w[i] = m.w[i] & n.w[i]; + return p; +} + +CandidateList similarTextHeuristicCandidates( const MetaTranslator *tor, + const char *text, + int maxCandidates ) +{ + TQValueList<int> scores; + CandidateList candidates; + CoMatrix cmTarget( text ); + int targetLen = tqstrlen( text ); + + TML all = tor->translatedMessages(); + TML::Iterator it; + + for ( it = all.begin(); it != all.end(); ++it ) { + if ( (*it).type() == MetaTranslatorMessage::Unfinished || + (*it).translation().isEmpty() ) + continue; + + TQString s = tor->toUnicode( (*it).sourceText(), (*it).utf8() ); + CoMatrix cm( s.latin1() ); + int delta = TQABS( (int) s.length() - targetLen ); + + /* + Here is the score formula. A comment above tqcontains a + discussion on a similar (but simpler) formula. + */ + int score = ( (intersection(cm, cmTarget).worth() + 1) << 10 ) / + ( reunion(cm, cmTarget).worth() + (delta << 1) + 1 ); + + if ( (int) candidates.count() == maxCandidates && + score > scores[maxCandidates - 1] ) + candidates.remove( candidates.last() ); + if ( (int) candidates.count() < maxCandidates && score >= 190 ) { + Candidate cand( s, (*it).translation() ); + + int i; + for ( i = 0; i < (int) candidates.count(); i++ ) { + if ( score >= scores[i] ) { + if ( score == scores[i] ) { + if ( candidates[i] == cand ) + goto continue_outer_loop; + } else { + break; + } + } + } + scores.insert( scores.at(i), score ); + candidates.insert( candidates.at(i), cand ); + } + continue_outer_loop: + ; + } + return candidates; +} |