Copy the KDE 3.5 branch to branches/trinity for new KDE 3.5 features.

BUG:215923


git-svn-id: svn://anonsvn.kde.org/home/kde/branches/trinity/kdepim@1054174 283d02a7-25f6-0310-bc7c-ecb5cbfe19da

1 files changed, 617 insertions, 0 deletions

diff --git a/mimelib/token.cpp b/mimelib/token.cpp
new file mode 100644
index 000000000..654d764f7
--- /dev/null
+++ b/mimelib/token.cpp
@@ -0,0 +1,617 @@
+//=============================================================================
+// File:       token.cpp
+// Contents:   Definitions for DwTokenizer, DwRfc822Tokenizer
+// Maintainer: Doug Sauder <dwsauder@fwb.gulf.net>
+// WWW:        http://www.fwb.gulf.net/~dwsauder/mimepp.html
+//
+// Copyright (c) 1996, 1997 Douglas W. Sauder
+// All rights reserved.
+//
+// IN NO EVENT SHALL DOUGLAS W. SAUDER BE LIABLE TO ANY PARTY FOR DIRECT,
+// INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF
+// THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF DOUGLAS W. SAUDER
+// HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// DOUGLAS W. SAUDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT
+// NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+// PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS"
+// BASIS, AND DOUGLAS W. SAUDER HAS NO OBLIGATION TO PROVIDE MAINTENANCE,
+// SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+//
+//=============================================================================
+
+#define DW_IMPLEMENTATION
+
+#include <mimelib/config.h>
+#include <mimelib/debug.h>
+#include <assert.h>
+#include <ctype.h>
+#include <mimelib/string.h>
+#include <mimelib/token.h>
+
+
+std::ostream* DwTokenizer::mDebugOut = 0;
+
+
+DwTokenizer::DwTokenizer(const DwString& aStr)
+  : mString(aStr)
+{
+    mTokenStart  = 0;
+    mTokenLength = 0;
+    mNextStart   = 0;
+    mTkType      = eTkError;
+}
+
+
+DwTokenizer::DwTokenizer(const char* aCStr)
+  : mString(aCStr)
+{
+    mTokenStart  = 0;
+    mTokenLength = 0;
+    mNextStart   = 0;
+    mTkType      = eTkError;
+}
+
+
+DwTokenizer::~DwTokenizer()
+{
+}
+
+
+void DwTokenizer::StripDelimiters()
+{
+    if (mTokenLength < 2) return;
+    // const ref -- avoids copy on write when using operator[]
+    const DwString& token = mToken;
+    switch (mTkType) {
+    case eTkQuotedString:
+        if (token[0] == '"') {
+            mToken = mToken.substr(1);
+            ++mTokenStart;
+            --mTokenLength;
+        }
+        if (mTokenLength > 0 && token[mTokenLength-1] == '"') {
+            mToken = mToken.substr(0, mTokenLength-1);
+            --mTokenLength;
+        }
+        break;
+    case eTkDomainLiteral:
+        if (token[0] == '[') {
+            mToken = mToken.substr(1);
+            ++mTokenStart;
+            --mTokenLength;
+        }
+        if (mTokenLength > 0 && token[mTokenLength-1] == ']') {
+            mToken = mToken.substr(0, mTokenLength-1);
+            --mTokenLength;
+        }
+        break;
+    case eTkComment:
+        if (token[0] == '(') {
+            mToken = mToken.substr(1);
+            ++mTokenStart;
+            --mTokenLength;
+        }
+        if (mTokenLength > 0 && token[mTokenLength-1] == ')') {
+            mToken = mToken.substr(0, mTokenLength-1);
+            --mTokenLength;
+        }
+        break;
+    }
+}
+
+
+void DwTokenizer::ParseQuotedString()
+{
+    size_t pos = mTokenStart;
+    while (1) {
+        ++pos;
+        if (pos >= mString.length()) {
+            // Ran out of string
+            mTokenLength = 0;
+            mToken = "";
+            mNextStart = pos;
+            mTkType = eTkError;
+            break;
+        }
+        else if (mString[pos] == '\\') {
+            // Quoted character
+            ++pos;
+            if (pos >= mString.length()) {
+                // Ran out of string
+                mTokenLength = 0;
+                mToken = "";
+                mNextStart = pos;
+                mTkType = eTkError;
+                break;
+            }
+        }
+        else if (mString[pos] == '"') {
+            // End of quoted string
+            ++pos;
+            mTokenLength = pos - mTokenStart;
+            mToken = mString.substr(mTokenStart, mTokenLength);
+            mNextStart = pos;
+            break;
+        }
+    }
+}
+
+
+void DwTokenizer::ParseComment()
+{
+    size_t pos = mTokenStart;
+    int level = 1;
+    while (1) {
+        ++pos;
+        if (pos >= mString.length()) {
+            // Ran out of string
+            mTokenLength = 0;
+            mToken = "";
+            mNextStart = pos;
+            mTkType = eTkError;
+            break;
+        }
+        else if (mString[pos] == '\\') {
+            // Quoted character
+            ++pos;
+            if (pos >= mString.length()) {
+                // Ran out of string
+                mTokenLength = 0;
+                mToken = "";
+                mNextStart = pos;
+                mTkType = eTkError;
+                break;
+            }
+        }
+        else if (mString[pos] == ')') {
+            --level;
+            if (level == 0) {
+                // End of comment
+                ++pos;
+                mTokenLength = pos - mTokenStart;
+                mToken = mString.substr(mTokenStart, mTokenLength);
+                mNextStart = pos;
+                break;
+            }
+        }
+        else if (mString[pos] == '(') {
+            ++level;
+        }
+    }
+}
+
+
+void DwTokenizer::ParseDomainLiteral()
+{
+    size_t pos = mTokenStart;
+    while (1) {
+        ++pos;
+        if (pos >= mString.length()) {
+            // Ran out of string
+            mTokenLength = 0;
+            mToken = "";
+            mNextStart = pos;
+            mTkType = eTkError;
+            break;
+        }
+        else if (mString[pos] == '\\') {
+            // Quoted character
+            ++pos;
+            if (pos >= mString.length()) {
+                // Ran out of string
+                mTokenLength = 0;
+                mToken = "";
+                mNextStart = pos;
+                mTkType = eTkError;
+                break;
+            }
+        }
+        else if (mString[pos] == ']') {
+            // End of domain literal
+            ++pos;
+            mTokenLength = pos - mTokenStart;
+            mToken = mString.substr(mTokenStart, mTokenLength);
+            mNextStart = pos;
+            break;
+        }
+    }
+}
+
+
+void DwTokenizer::PrintToken(std::ostream* aOut)
+{
+    if (!aOut) return;
+    const char* type = 0;
+    switch (mTkType) {
+    case eTkError:
+        type = "error          ";
+        break;
+    case eTkNull:
+        type = "null           ";
+        break;
+    case eTkSpecial:
+        type = "special        ";
+        break;
+    case eTkAtom:
+        type = "atom           ";
+        break;
+    case eTkComment:
+        type = "comment        ";
+        break;
+    case eTkQuotedString:
+        type = "quoted string  ";
+        break;
+    case eTkDomainLiteral:
+        type = "domain literal ";
+        break;
+    case eTkTspecial:
+        type = "tspecial       ";
+        break;
+    case eTkToken:
+        type = "token          ";
+        break;
+    default:
+        type = "unknown        ";
+        break;
+    }
+    *aOut << type << mToken << '\n';
+}
+
+
+static inline bool isspecialorspaceorcntrl( int c ) 
+{
+  switch ( c ) {
+     case '(':
+     case ')':
+     case '<':
+     case '>':
+     case '@':
+     case ',':
+     case ';':
+     case ':':
+     case '\\':
+     case '"':
+     case '.':
+     case '[':
+     case ']':
+     // isspace()
+     case ' ':
+         return true;
+     //case '\r': included in iscntrl()
+     //case '\f': included in iscntrl()
+     //case '\t': included in iscntrl()
+     //case '\n': included in iscntrl()
+     //case '\v': included in iscntrl()
+     // iscntrl()
+     default:
+         return ( (c >= 0 && c <= 15) || (c >= 17 && c <= 31) );
+  }
+}
+
+static inline bool isnotspaceorcntrl( int c ) 
+{
+  switch ( c ) {
+     // isspace()
+     case ' ':
+     //case '\r': included in iscntrl()
+     //case '\f': included in iscntrl()
+     //case '\t': included in iscntrl()
+     //case '\n': included in iscntrl()
+     //case '\v': included in iscntrl()
+     // iscntrl()
+         return false;
+     default:
+         return !( (c >= 0 && c <= 15) || (c >= 17 && c <= 31) );
+  }
+}
+
+DwRfc822Tokenizer::DwRfc822Tokenizer(const DwString& aStr)
+  : DwTokenizer(aStr)
+{
+    ParseToken();
+}
+
+
+DwRfc822Tokenizer::DwRfc822Tokenizer(const char* aCStr)
+  : DwTokenizer(aCStr)
+{
+    ParseToken();
+}
+
+
+DwRfc822Tokenizer::~DwRfc822Tokenizer()
+{
+}
+
+
+int DwRfc822Tokenizer::Restart()
+{
+    mNextStart = 0;
+    ParseToken();
+    return mTkType;
+}
+
+
+int DwRfc822Tokenizer::operator ++ ()
+{
+    ParseToken();
+    return mTkType;
+}
+
+
+void DwRfc822Tokenizer::ParseToken()
+{
+    // Assume the field body has already been extracted.  That is, we don't
+    // have to watch for the end of the field body or folding.  We just
+    // treat any CRs or LFs as white space.
+    mTokenStart = mNextStart;
+    mTokenLength = 0;
+    mTkType = eTkNull;
+    // Skip leading space.  Also, since control chars are not permitted
+    // in atoms, skip these, too.
+    while (1) {
+        if (mTokenStart >= mString.length()) {
+            return;
+        }
+        if (isnotspaceorcntrl(mString[mTokenStart]))
+            break;
+        ++mTokenStart;
+    }
+    char ch = mString[mTokenStart];
+    switch (ch) {
+        // Quoted string
+        case '"':
+            mTkType = eTkQuotedString;
+            ParseQuotedString();
+        break;
+        // Comment
+        case '(':
+            mTkType = eTkComment;
+            ParseComment();
+        break;
+        // Domain literal
+        case '[':
+            mTkType = eTkDomainLiteral;
+            ParseDomainLiteral();
+        break;
+        // Special
+        case ')':
+        case '<':
+        case '>':
+        case '@':
+        case ',':
+        case ';':
+        case ':':
+        case '\\':
+        case '.':
+        case ']':
+            mTkType = eTkSpecial;
+            mTokenLength = 1;
+            mToken = mString.substr(mTokenStart, 1);
+            mNextStart = mTokenStart + 1;
+        break;
+        default:
+            mTkType = eTkAtom;
+            ParseAtom();
+        break;
+    }
+    if (mDebugOut) PrintToken(mDebugOut);
+}
+
+
+void DwRfc822Tokenizer::ParseAtom()
+{
+    size_t pos = mTokenStart;
+    while (1) {
+        ++pos;
+        char ch = (pos < mString.length()) ? mString[pos] : (char) 0;
+        if (pos >= mString.length()
+            || isspecialorspaceorcntrl(ch)) {
+
+            mTokenLength = pos - mTokenStart;
+            mToken = mString.substr(mTokenStart, mTokenLength);
+            mNextStart = pos;
+            break;
+        }
+    }
+}
+
+static inline bool istspecialorspaceorcntrl( int c ) 
+{
+  switch ( c ) {
+     case '(':
+     case ')':
+     case '<':
+     case '>':
+     case '@':
+     case ',':
+     case ';':
+     case ':':
+     case '\\':
+     case '"':
+     case '/':
+     case '[':
+     case ']':
+     case '?':
+     case '=':
+     // isspace()
+     case ' ':
+         return true;
+     //case '\r': included in iscntrl()
+     //case '\f': included in iscntrl()
+     //case '\t': included in iscntrl()
+     //case '\n': included in iscntrl()
+     //case '\v': included in iscntrl()
+     // iscntrl()
+     default:
+        return ( ( c >= 0 && c <= 15) || (c >= 17 && c <= 31) );
+  }
+ }
+
+DwRfc1521Tokenizer::DwRfc1521Tokenizer(const DwString& aStr)
+  : DwTokenizer(aStr)
+{
+    ParseToken();
+}
+
+
+DwRfc1521Tokenizer::DwRfc1521Tokenizer(const char* aCStr)
+  : DwTokenizer(aCStr)
+{
+    ParseToken();
+}
+
+
+DwRfc1521Tokenizer::~DwRfc1521Tokenizer()
+{
+}
+
+
+int DwRfc1521Tokenizer::Restart()
+{
+    mNextStart = 0;
+    ParseToken();
+    return mTkType;
+}
+
+
+int DwRfc1521Tokenizer::operator ++ ()
+{
+    ParseToken();
+    return mTkType;
+}
+
+
+void DwRfc1521Tokenizer::ParseToken()
+{
+    // Assume the field body has already been extracted.  That is, we don't
+    // have to watch for the end of the field body or folding.  We just
+    // treat any CRs or LFs as white space.
+    mTokenStart = mNextStart;
+    mTokenLength = 0;
+    mTkType = eTkNull;
+    // Skip leading space.  Also, since control chars are not permitted
+    // in atoms, skip these, too.
+    while (1) {
+        if (mTokenStart >= mString.length()) {
+            return;
+        }
+        if (isnotspaceorcntrl(mString[mTokenStart]))
+            break;
+        ++mTokenStart;
+    }
+    char ch = mString[mTokenStart];
+    switch (ch) {
+        // Quoted string
+        case '"':
+            mTkType = eTkQuotedString;
+            ParseQuotedString();
+        break;
+        // Comment
+        case '(':
+            mTkType = eTkComment;
+            ParseComment();
+        break;
+        // Domain literal
+        case '[':
+            mTkType = eTkDomainLiteral;
+            ParseDomainLiteral();
+        break;
+        // Special
+        case ')':
+        case '<':
+        case '>':
+        case '@':
+        case ',':
+        case ';':
+        case ':':
+        case '\\':
+        case '/':
+        case ']':
+        case '?':
+        case '=':
+            mTkType = eTkTspecial;
+            mTokenLength = 1;
+            mToken = mString.substr(mTokenStart, 1);
+            mNextStart = mTokenStart + 1;
+        break;
+        default:
+            mTkType = eTkToken;
+            ParseAtom();
+        break;
+    }
+    if (mDebugOut) PrintToken(mDebugOut);
+}
+
+
+void DwRfc1521Tokenizer::ParseAtom()
+{
+    size_t pos = mTokenStart;
+    while (1) {
+        ++pos;
+        char ch = (pos < mString.length()) ? mString[pos] : (char) 0;
+        if (pos >= mString.length()
+            || istspecialorspaceorcntrl(ch)) {
+
+            mTokenLength = pos - mTokenStart;
+            mToken = mString.substr(mTokenStart, mTokenLength);
+            mNextStart = pos;
+            break;
+        }
+    }
+}
+
+
+DwTokenString::DwTokenString(const DwString& aStr)
+  : mString(aStr)
+{
+    mTokensStart  = 0;
+    mTokensLength = 0;
+}
+
+
+DwTokenString::~DwTokenString()
+{
+}
+
+
+void DwTokenString::SetFirst(const DwTokenizer& aTkzr)
+{
+    switch (aTkzr.Type()) {
+    case eTkError:
+    case eTkNull:
+        mTokensStart  = aTkzr.mTokenStart;
+        mTokensLength = 0;
+        break;
+    case eTkComment:
+    case eTkDomainLiteral:
+    case eTkQuotedString:
+    case eTkSpecial:
+    case eTkAtom:
+    case eTkTspecial:
+    case eTkToken:
+        mTokensStart  = aTkzr.mTokenStart;
+        mTokensLength = aTkzr.mTokenLength;
+        break;
+    }
+    mTokens = mString.substr(mTokensStart, mTokensLength);
+}
+
+
+void DwTokenString::SetLast(const DwTokenizer& aTkzr)
+{
+    assert(aTkzr.mTokenStart >= mTokensStart);
+    if (aTkzr.mTokenStart < mTokensStart) return;
+    mTokensLength = aTkzr.mTokenStart + aTkzr.mTokenLength - mTokensStart;
+    mTokens = mString.substr(mTokensStart, mTokensLength);
+}
+
+
+void DwTokenString::ExtendTo(const DwTokenizer& aTkzr)
+{
+    assert(aTkzr.mTokenStart >= mTokensStart);
+    if (aTkzr.mTokenStart < mTokensStart) return;
+    mTokensLength = aTkzr.mTokenStart - mTokensStart;
+    mTokens = mString.substr(mTokensStart, mTokensLength);
+}