Copy the KDE 3.5 branch to branches/trinity for new KDE 3.5 features.

BUG:215923


git-svn-id: svn://anonsvn.kde.org/home/kde/branches/trinity/kdewebdev@1054174 283d02a7-25f6-0310-bc7c-ecb5cbfe19da

1 files changed, 703 insertions, 0 deletions

diff --git a/klinkstatus/src/engine/linkchecker.cpp b/klinkstatus/src/engine/linkchecker.cpp
new file mode 100644
index 00000000..bcc503ad
--- /dev/null
+++ b/klinkstatus/src/engine/linkchecker.cpp
@@ -0,0 +1,703 @@
+/***************************************************************************
+ *   Copyright (C) 2004 by Puto Moura                                      *
+ *   mojo@localhost.localdomain                                            *
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
+ *   GNU General Public License for more details.                          *
+ *                                                                         *
+ *   You should have received a copy of the GNU General Public License     *
+ *   along with this program; if not, write to the                         *
+ *   Free Software Foundation, Inc.,                                       *
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.             *
+ ***************************************************************************/
+#include "linkchecker.h"
+#include "searchmanager.h"
+#include "../utils/utils.h"
+#include "../parser/htmlparser.h"
+
+#include <qstring.h>
+#include <qtimer.h>
+#include <qtextcodec.h> 
+#include <qcstring.h>
+
+#include <kio/netaccess.h>
+#include <kio/global.h>
+#include <kio/job.h>
+#include <kio/scheduler.h>
+#include <kio/slave.h>
+#include <kmimetype.h>
+#include <kapplication.h>
+#include <klocale.h>
+#include <khtml_part.h>
+#include <dom/html_misc.h>
+#include <dom/dom_node.h>
+#include <dom/dom_string.h>
+
+
+int LinkChecker::count_ = 0;
+
+LinkChecker::LinkChecker(LinkStatus* linkstatus, int time_out,
+                         QObject *parent, const char *name)
+        : QObject(parent, name), search_manager_(0), 
+        linkstatus_(linkstatus), t_job_(0), time_out_(time_out), checker_(0), document_charset_(), 
+        redirection_(false), header_checked_(false), finnished_(false), 
+        parsing_(false), is_charset_checked_(false), has_defined_charset_(false)
+{
+    Q_ASSERT(linkstatus_);
+    Q_ASSERT(!linkstatus_->checked());
+
+    kdDebug(23100) << endl << ++count_ << ": " << "Checking " << linkstatus_->absoluteUrl().url() << endl;
+}
+
+LinkChecker::~LinkChecker()
+{}
+
+void LinkChecker::setSearchManager(SearchManager* search_manager)
+{
+    Q_ASSERT(search_manager);
+    search_manager_ = search_manager;
+}
+
+void LinkChecker::check()
+{
+    Q_ASSERT(!finnished_);
+
+    KURL url(linkStatus()->absoluteUrl());
+    Q_ASSERT(url.isValid());
+
+    if(url.hasRef()) {
+        KMimeType::Ptr mimeType = KMimeType::findByURL(url);
+        if(mimeType->is("text/html") || mimeType->is("application/xml")) {
+            checkRef();
+            return;
+        }
+    }
+        
+    t_job_ = KIO::get(url, false, false);
+
+    t_job_->addMetaData("PropagateHttpHeader", "true"); // to have the http header
+
+    if (linkstatus_->parent()) {
+        t_job_->addMetaData("referrer", linkstatus_->parent()->absoluteUrl().prettyURL());
+    }
+
+    if(search_manager_->sendIdentification())
+    {
+        t_job_->addMetaData("SendUserAgent", "true");
+        t_job_->addMetaData("UserAgent", search_manager_->userAgent());
+    }
+    else
+        t_job_->addMetaData("SendUserAgent", "false");
+
+
+    QObject::connect(t_job_, SIGNAL(data(KIO::Job *, const QByteArray &)),
+                        this, SLOT(slotData(KIO::Job *, const QByteArray &)));
+    QObject::connect(t_job_, SIGNAL(mimetype(KIO::Job *, const QString &)),
+                        this, SLOT(slotMimetype(KIO::Job *, const QString &)));
+    QObject::connect(t_job_, SIGNAL(result(KIO::Job *)),
+                        this, SLOT(slotResult(KIO::Job *)));
+    QObject::connect(t_job_, SIGNAL(redirection(KIO::Job *, const KURL &)),
+                        this, SLOT(slotRedirection(KIO::Job *, const KURL &)));
+
+    QTimer::singleShot( time_out_ * 1000, this, SLOT(slotTimeOut()) );
+
+    t_job_->setInteractive(false);    
+}
+
+void LinkChecker::slotTimeOut()
+{
+    if(!finnished_ && !parsing_)
+    {
+        kdDebug(23100) << "timeout: " << linkstatus_->absoluteUrl().url() << endl;
+        if(t_job_ && t_job_->slave())
+            kdDebug(23100) << " - " << t_job_->slave() << "/" <<  t_job_->slave()->slave_pid() << endl;
+        else
+            kdDebug(23100) << endl;
+
+        
+//         Q_ASSERT(t_job_); // can happen: e.g. bad result signal
+        if(t_job_->error() != KIO::ERR_USER_CANCELED)
+        {
+            linkstatus_->setErrorOccurred(true);
+            linkstatus_->setChecked(true);
+            linkstatus_->setError(i18n("Timeout"));
+            linkstatus_->setStatus(LinkStatus::TIMEOUT);
+
+            killJob();
+            finnish();
+        }
+    }
+}
+
+void LinkChecker::slotMimetype (KIO::Job* /*job*/, const QString &type)
+{
+    if(finnished_)
+        return;
+
+//     kdDebug(23100) <<  "LinkChecker::slotMimetype:" << type << "-> " << linkstatus_->absoluteUrl().url() 
+//             << " - " << t_job_->slave() << "/" <<  t_job_->slave()->slave_pid() << endl;
+
+    Q_ASSERT(t_job_);
+
+    LinkStatus* ls = 0;
+/*    if(redirection_)
+        ls = linkStatus()->redirection();
+    else*/
+        ls = linkstatus_;
+    Q_ASSERT(ls);
+
+    ls->setMimeType(type);
+    KURL url = ls->absoluteUrl();
+
+    // we doesn't do nothing if file is http or https because we need the header
+    // which is only available in the data response
+    if(!t_job_->error()) // if a error happened let result() handle that
+    {
+        if(ls->onlyCheckHeader())
+        {
+            //kdDebug(23100) <<  "only check header: " << ls->absoluteUrl().prettyURL() << endl;
+
+            // file is OK (http can have an error page though job->error() is false)
+            if(!url.protocol().startsWith("http"))
+            {
+                ls->setStatusText("OK");
+                ls->setStatus(LinkStatus::SUCCESSFULL);
+                
+                killJob();                
+                finnish();
+            }
+        }
+        else // !ls->onlyCheckHeader()
+        {
+            //kdDebug(23100) <<  "NOT only check header: " << ls->absoluteUrl().prettyURL() << endl;
+
+            // file is OK (http can have an error page though job->error() is false)
+            if(!url.protocol().startsWith("http")) // if not, it have to go trough slotData to get the http header
+            {
+                // it's not an html page, so we don't want the file content
+                if(type != "text/html"/* && type != "text/plain"*/)
+                {
+                    //kdDebug(23100) <<  "mimetype: " << type << endl;
+                    ls->setStatusText("OK");
+                    ls->setStatus(LinkStatus::SUCCESSFULL);
+                    
+                    killJob();                    
+                    finnish();
+                }
+            }
+        }
+    }
+}
+
+void LinkChecker::slotData(KIO::Job* /*job*/, const QByteArray& data)
+{   
+    if(finnished_)
+        return;
+
+    kdDebug(23100) <<  "LinkChecker::slotData -> " << linkstatus_->absoluteUrl().url() 
+            << " - " << t_job_->slave() << "/" <<  t_job_->slave()->slave_pid()  << endl;
+    
+    Q_ASSERT(t_job_);
+    
+    LinkStatus* ls = 0;
+/*    if(redirection_)
+        ls = linkStatus()->redirection();
+    else*/
+        ls = linkstatus_;
+    Q_ASSERT(ls);
+
+    KURL url = ls->absoluteUrl();
+
+    if(!t_job_->error())
+    {
+        if(ls->onlyCheckHeader())
+        {
+            Q_ASSERT(header_checked_ == false);
+            // the job should have been killed in slotMimetype
+            Q_ASSERT(url.protocol() == "http" || url.protocol() == "https");
+
+            // get the header and quit
+            if(url.protocol().startsWith("http"))
+            {
+                // get the header
+                ls->setHttpHeader(getHttpHeader(t_job_));
+
+                if(t_job_->isErrorPage())
+                    ls->setIsErrorPage(true);
+
+                if(header_checked_)
+                {
+                    killJob();
+                    linkstatus_->setStatus(getHttpStatus());
+                    linkstatus_->setChecked(true);
+                    finnish();
+                    return;
+                }
+            }
+        }
+        else
+        {
+            if(url.protocol().startsWith("http"))
+            {
+                if(!header_checked_)
+                {
+                    ls->setHttpHeader(getHttpHeader(t_job_));                    
+                }
+                if(ls->mimeType() != "text/html" && header_checked_)
+                {
+                    //kdDebug(23100) <<  "mimetype of " << ls->absoluteUrl().prettyURL() << ": " << ls->mimeType() << endl;
+                    ls->setStatus(getHttpStatus());
+                    killJob();
+                    finnish(); // if finnish is called before kill what you get is a segfault, don't know why
+                    return;
+                }
+                else if(t_job_->isErrorPage() && header_checked_)
+                {
+                    //kdDebug(23100) <<  "ERROR PAGE" << endl;
+                    ls->setIsErrorPage(true);
+                    ls->setStatus(getHttpStatus());
+                    killJob();
+                    finnish();
+                    return;
+                }
+            }
+            else
+            {
+                Q_ASSERT(ls->mimeType() == "text/html");
+            }
+            if(!is_charset_checked_) 
+                findDocumentCharset(data);
+            
+            QTextCodec* codec = 0;
+            if(has_defined_charset_) 
+                codec = QTextCodec::codecForName(document_charset_);
+            if(!codec)
+                codec = QTextCodec::codecForName("iso8859-1"); // default
+            
+            doc_html_ += codec->toUnicode(data);
+        }
+    }
+}
+
+void LinkChecker::findDocumentCharset(QString const& doc)
+{
+    Q_ASSERT(!is_charset_checked_);
+    
+    is_charset_checked_ = true; // only check the first stream of data
+                    
+    if(header_checked_)
+        document_charset_ = linkstatus_->httpHeader().charset();
+
+    // try to look in the meta elements                    
+    if(document_charset_.isNull() || document_charset_.isEmpty()) 
+        document_charset_ = HtmlParser::findCharsetInMetaElement(doc);
+    
+    if(!document_charset_.isNull() && !document_charset_.isEmpty())
+        has_defined_charset_ = true;
+}
+
+// only comes here if an error happened or in case of a clean html page
+// if onlyCheckHeader is false
+void LinkChecker::slotResult(KIO::Job* /*job*/)
+{
+    if(finnished_)
+        return;
+
+    kdDebug(23100) <<  "LinkChecker::slotResult -> " << linkstatus_->absoluteUrl().url()  << endl;
+
+    Q_ASSERT(t_job_);
+    if(!t_job_)
+        return;
+    
+    if(redirection_) {
+        if(!processRedirection(redirection_url_)) {
+            t_job_ = 0;
+            linkstatus_->setChecked(true);
+            finnish();
+            return;
+        }
+    }
+            
+    KIO::TransferJob* job = t_job_;
+    t_job_ = 0;
+
+    emit jobFinnished(this);
+
+    if(job->error() == KIO::ERR_USER_CANCELED)
+    {
+        // FIXME This can happen! If the job is non interactive...
+        kdWarning(23100) << endl << "Job killed quietly, yet signal result was emited..." << endl;
+        kdDebug(23100) << linkstatus_->toString() << endl;
+        finnish();
+        return;
+    }
+
+    LinkStatus* ls = 0;
+    if(redirection_)
+        ls = linkStatus()->redirection();
+    else
+        ls = linkstatus_;
+    Q_ASSERT(ls);
+
+    if(!(!ls->onlyCheckHeader() ||
+          job->error() ||
+          !header_checked_))
+        kdWarning(23100) << ls->toString() << endl;
+
+    Q_ASSERT(!ls->onlyCheckHeader() || job->error() || !header_checked_);
+
+    if(ls->isErrorPage())
+        kdWarning(23100) << "\n\n" << ls->toString() << endl << endl;
+
+    Q_ASSERT(!job->isErrorPage());
+
+    if(job->error())
+    {
+        kdDebug(23100) << "Job error: " <<  job->errorString() << endl;
+        kdDebug(23100) << "Job error code: " <<  job->error() << endl;
+
+        if(job->error() == KIO::ERR_IS_DIRECTORY)
+        {
+            ls->setStatusText("OK");
+            ls->setStatus(LinkStatus::SUCCESSFULL);
+        }
+        else
+        {
+            ls->setErrorOccurred(true);
+            if(job->error() == KIO::ERR_SERVER_TIMEOUT)
+                ls->setStatus(LinkStatus::TIMEOUT);
+            else
+                ls->setStatus(LinkStatus::BROKEN);
+
+            if(job->errorString().isEmpty())
+                kdWarning(23100) << "\n\nError string is empty, error = " << job->error() << "\n\n\n";
+            if(job->error() != KIO::ERR_NO_CONTENT) 
+                ls->setError(job->errorString());
+            else
+                ls->setError(i18n("No Content"));
+        }
+    }
+
+    else
+    {
+        if(!ls->absoluteUrl().protocol().startsWith("http")) {
+            ls->setStatusText("OK");
+            ls->setStatus(LinkStatus::SUCCESSFULL);
+           }
+        else
+        {
+            if(!header_checked_)
+            {
+                kdDebug(23100) <<  "\n\nheader not received... checking again...\n\n\n";
+                //check again
+                check();
+                return;
+            }
+            Q_ASSERT(header_checked_);
+            
+            ls->setStatus(getHttpStatus());
+        }
+
+        if(!doc_html_.isNull() && !doc_html_.isEmpty())
+        {
+            ls->setDocHtml(doc_html_);
+
+            parsing_ = true;
+            HtmlParser parser(doc_html_);
+
+            if(parser.hasBaseUrl())
+                ls->setBaseURI(KURL(parser.baseUrl().url()));
+            if(parser.hasTitle())
+                ls->setHtmlDocTitle(parser.title().attributeTITLE());
+            
+            ls->setChildrenNodes(parser.nodes());
+            parsing_ = false;
+        }
+    }
+    finnish();
+}
+
+
+void LinkChecker::slotRedirection (KIO::Job* /*job*/, const KURL &url)
+{
+    kdDebug(23100) <<  "LinkChecker::slotRedirection -> " << 
+            linkstatus_->absoluteUrl().url()  << " -> " << url.url() << endl;
+//             << " - " << t_job_->slave() << "/" <<  t_job_->slave()->slave_pid() << endl;
+    
+    redirection_ = true;
+    redirection_url_ = url;
+}
+
+bool LinkChecker::processRedirection(KURL const& toUrl)
+{
+    if(finnished_)
+        return true;
+
+    kdDebug(23100) <<  "LinkChecker::processRedirection -> " << linkstatus_->absoluteUrl().url() << " -> " << toUrl.url() << endl;
+
+    Q_ASSERT(t_job_);
+    Q_ASSERT(linkstatus_->absoluteUrl().protocol().startsWith("http"));
+    Q_ASSERT(redirection_);
+
+    linkstatus_->setHttpHeader(getHttpHeader(t_job_, false));
+    linkstatus_->setIsRedirection(true);
+    linkstatus_->setStatusText("redirection");
+    linkstatus_->setStatus(LinkStatus::HTTP_REDIRECTION);
+    linkstatus_->setChecked(true);
+    
+    LinkStatus* ls_red = new LinkStatus(*linkstatus_);
+    ls_red->setAbsoluteUrl(toUrl);
+    ls_red->setRootUrl(linkstatus_->rootUrl());
+
+    if(!linkstatus_->onlyCheckHeader())
+        ls_red->setOnlyCheckHeader(false);
+
+    linkstatus_->setRedirection(ls_red);
+    ls_red->setParent(linkstatus_);
+    ls_red->setOriginalUrl(toUrl.url());
+
+    Q_ASSERT(search_manager_);
+
+    if(search_manager_->localDomain(ls_red->absoluteUrl()))
+        ls_red->setExternalDomainDepth(-1);
+    else
+    {
+        if(search_manager_->localDomain(linkstatus_->absoluteUrl()))
+            ls_red->setExternalDomainDepth(linkstatus_->externalDomainDepth() + 1);
+        else
+            ls_red->setExternalDomainDepth(linkstatus_->externalDomainDepth());
+    }
+
+    if(!toUrl.isValid() || search_manager_->existUrl(toUrl, linkstatus_->absoluteUrl()))
+    {
+        ls_red->setChecked(false);
+        return false;
+    }
+    else
+    {
+        ls_red->setChecked(true);
+        return true;
+    }
+}
+
+void LinkChecker::finnish()
+{
+    Q_ASSERT(!t_job_);
+
+    if(!finnished_)
+    {
+        kdDebug(23100) <<  "LinkChecker::finnish -> " << linkstatus_->absoluteUrl().url() << endl;
+
+        finnished_ = true;
+
+        if(redirection_)
+            Q_ASSERT(linkstatus_->checked());
+        else
+            linkstatus_->setChecked(true);
+
+        emit transactionFinished(linkstatus_, this);
+    }
+}
+
+HttpResponseHeader LinkChecker::getHttpHeader(KIO::Job* /*job*/, bool remember_check)
+{
+    //kdDebug(23100) <<  "LinkChecker::getHttpHeader -> " << linkstatus_->absoluteUrl().url() << endl;
+    
+    Q_ASSERT(!finnished_);
+    Q_ASSERT(t_job_);
+
+    QString header_string = t_job_->queryMetaData("HTTP-Headers");
+    //    Q_ASSERT(!header_string.isNull() && !header_string.isEmpty());
+//     kdDebug(23100) << "HTTP header: " << endl << header_string << endl;
+//     kdDebug(23100) << "Keys: " << HttpResponseHeader(header_string).keys() << endl;
+//     kdDebug(23100) << "Content-type: " << HttpResponseHeader(header_string).contentType() << endl;
+//     kdDebug(23100) << "Content-type: " << HttpResponseHeader(header_string).value("content-type") << endl;
+
+    if(header_string.isNull() || header_string.isEmpty())
+    {
+        header_checked_ = false;
+        kdWarning(23100) <<  "header_string.isNull() || header_string.isEmpty(): "
+        << linkstatus_->toString()  << endl;
+    }
+    else if(remember_check)
+        header_checked_ = true;
+
+    return HttpResponseHeader(header_string);
+}
+
+void LinkChecker::checkRef()
+{
+    KURL url(linkStatus()->absoluteUrl());
+    Q_ASSERT(url.hasRef());
+    
+    QString ref = url.ref();
+    if(ref == "" || ref == "top") {
+        linkstatus_->setStatusText("OK");
+        linkstatus_->setStatus(LinkStatus::SUCCESSFULL);
+        finnish();
+        return;
+    }
+
+    QString url_base;
+    LinkStatus const* ls_parent = 0;
+    int i_ref = -1;
+
+    if(linkStatus()->originalUrl().startsWith("#"))
+        ls_parent = linkStatus()->parent();
+
+    else
+    {
+        i_ref = url.url().find("#");
+        url_base = url.url().left(i_ref);
+        //kdDebug(23100) << "url_base: " << url_base << endl;
+
+        Q_ASSERT(search_manager_);
+
+        ls_parent = search_manager_->linkStatus(url_base);
+    }
+
+    if(ls_parent)
+        checkRef(ls_parent);
+    else
+    {
+        url = KURL::fromPathOrURL(url.url().left(i_ref));
+        checkRef(url);
+    }
+}
+
+void LinkChecker::checkRef(KURL const& url)
+{
+    Q_ASSERT(search_manager_);
+    
+    QString url_string = url.url();
+    KHTMLPart* html_part = search_manager_->htmlPart(url_string);
+    if(!html_part)
+    {
+        kdDebug() << "new KHTMLPart: " +  url_string << endl;
+
+        html_part = new KHTMLPart();
+        html_part->setOnlyLocalReferences(true);
+
+        QString tmpFile;
+        if(KIO::NetAccess::download(url, tmpFile, 0))
+        {
+            QString doc_html = FileManager::read(tmpFile);
+            html_part->begin();
+            html_part->write(doc_html);
+            html_part->end();
+
+            KIO::NetAccess::removeTempFile(tmpFile);
+        } 
+        else 
+        {
+            kdDebug(23100) <<  KIO::NetAccess::lastErrorString() << endl;
+        }
+
+        search_manager_->addHtmlPart(url_string, html_part);
+    }
+
+    if(hasAnchor(html_part, linkStatus()->absoluteUrl().ref()))
+    {
+        linkstatus_->setStatusText("OK");
+        linkstatus_->setStatus(LinkStatus::SUCCESSFULL);
+    }
+    else
+    {
+        linkstatus_->setErrorOccurred(true);
+        linkstatus_->setError(i18n( "Link destination not found." ));
+        linkstatus_->setStatus(LinkStatus::BROKEN);
+    }
+
+    finnish();
+}
+
+void LinkChecker::checkRef(LinkStatus const* linkstatus_parent)
+{
+    Q_ASSERT(search_manager_);
+
+    QString url_string = linkstatus_parent->absoluteUrl().url();
+    KHTMLPart* html_part = search_manager_->htmlPart(url_string);
+    if(!html_part)
+    {
+        kdDebug() << "new KHTMLPart: " +  url_string << endl;
+
+        html_part = new KHTMLPart();
+        html_part->setOnlyLocalReferences(true);
+
+        html_part->begin();
+        html_part->write(linkstatus_parent->docHtml());
+        html_part->end();
+
+        search_manager_->addHtmlPart(url_string, html_part);
+    }
+
+    if(hasAnchor(html_part, linkStatus()->absoluteUrl().ref()))
+    {
+        linkstatus_->setStatusText("OK");
+        linkstatus_->setStatus(LinkStatus::SUCCESSFULL);
+    }
+    else
+    {
+        linkstatus_->setErrorOccurred(true);
+        linkstatus_->setError(i18n( "Link destination not found." ));
+        linkstatus_->setStatus(LinkStatus::BROKEN);
+    }
+
+    finnish();
+}
+
+bool LinkChecker::hasAnchor(KHTMLPart* html_part, QString const& anchor)
+{
+    DOM::HTMLDocument htmlDocument = html_part->htmlDocument();
+    DOM::HTMLCollection anchors = htmlDocument.anchors();
+
+    DOM::DOMString name_ref(anchor);
+    Q_ASSERT(!name_ref.isNull());
+
+    DOM::Node node = anchors.namedItem(name_ref);
+    if(node.isNull())
+    {
+        node = htmlDocument.getElementById(name_ref);
+    }
+
+    if(!node.isNull())
+        return true;
+    else
+        return false;
+}
+
+void LinkChecker::killJob()
+{
+    if(!t_job_)
+        return;
+
+    KIO::TransferJob* aux = t_job_;
+    t_job_ = 0;
+    aux->disconnect(this);
+    aux->kill(true); // quietly   
+}
+
+LinkStatus::Status LinkChecker::getHttpStatus() const
+{
+    QString status_code = QString::number(linkstatus_->httpHeader().statusCode());
+    
+    if(status_code[0] == '2')
+        return LinkStatus::SUCCESSFULL;
+    else if(status_code[0] == '3')
+        return LinkStatus::HTTP_REDIRECTION;
+    else if(status_code[0] == '4')
+        return LinkStatus::HTTP_CLIENT_ERROR;
+    else if(status_code[0] == '5')
+        return LinkStatus::HTTP_SERVER_ERROR;
+    else
+        return LinkStatus::UNDETERMINED;
+}
+
+#include "linkchecker.moc"