path: root/src/z3950connection.cpp

/***************************************************************************
    copyright            : (C) 2005-2006 by Robby Stephenson
    email                : $EMAIL
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This file has been modified to match the requirements of KBibTeX.     *
 *   In case of problems or bugs arising from this implementation, please  *
 *   contact the KBibTeX team first.                                       *
 *                             Thomas Fischer <fischer@unix-ag.uni-kl.de>  *
 *                                                                         *
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of version 2 of the GNU General Public License as  *
 *   published by the Free Software Foundation;                            *
 *                                                                         *
 ***************************************************************************/

#include "z3950connection.h"
// #include "z3950fetcher.h"
#include "messagehandler.h"
#include "latin1literal.h"
#include <kdebug.h>
#include "iso5426converter.h"
#include "iso6937converter.h"

#include <qapplication.h>

#include <config.h>

#ifdef HAVE_YAZ
extern "C"
{
#include <yaz/zoom.h>
#include <yaz/marcdisp.h>
#include <yaz/yaz-version.h>
}
#endif

#include <klocale.h>

#include <qfile.h>

namespace
{
    static const size_t Z3950_DEFAULT_MAX_RECORDS = 20;
}

using KBibTeX::Z3950ResultFound;
using KBibTeX::Z3950Connection;

Z3950ResultFound::Z3950ResultFound( const QString& s ) : QCustomEvent( uid() )
        , m_result( QDeepCopy<QString>( s ) )
{
    ++Z3950Connection::resultsLeft;
}

Z3950ResultFound::~Z3950ResultFound()
{
    --Z3950Connection::resultsLeft;
}

class Z3950Connection::Private
{
public:
    Private() {}
#ifdef HAVE_YAZ
    ~Private()
    {
        ZOOM_options_destroy( conn_opt );
        ZOOM_connection_destroy( conn );
    };

    ZOOM_options conn_opt;
    ZOOM_connection conn;
#endif
};

int Z3950Connection::resultsLeft = 0;

// since the character set goes into a yaz api call
// I'm paranoid about user insertions, so just grab 64
// characters at most
Z3950Connection::Z3950Connection( QObject* fetcher,
                                  const QString& host,
                                  uint port,
                                  const QString& dbname,
                                  const QString& sourceCharSet,
                                  const QString& syntax,
                                  const QString& esn )
        : QThread()
        , d( new Private() )
        , m_connected( false )
        , m_aborted( false )
        , m_fetcher( fetcher )
        , m_host( QDeepCopy<QString>( host ) )
        , m_port( port )
        , m_dbname( QDeepCopy<QString>( dbname ) )
        , m_sourceCharSet( QDeepCopy<QString>( sourceCharSet.left( 64 ) ) )
        , m_syntax( QDeepCopy<QString>( syntax ) )
        , m_esn( QDeepCopy<QString>( esn ) )
        , m_start( 0 )
        , m_limit( Z3950_DEFAULT_MAX_RECORDS )
        , m_hasMore( false )
{
}

Z3950Connection::~Z3950Connection()
{
    m_connected = false;
    delete d;
    d = 0;
}

void Z3950Connection::reset()
{
    m_start = 0;
    m_limit = Z3950_DEFAULT_MAX_RECORDS;
}

void Z3950Connection::setQuery( const QString& query_, unsigned int numHits )
{
    m_pqn = QDeepCopy<QString>( query_ );
    m_limit = Z3950_DEFAULT_MAX_RECORDS < numHits ? Z3950_DEFAULT_MAX_RECORDS : numHits;
}

void Z3950Connection::setUserPassword( const QString& user_, const QString& pword_ )
{
    m_user = QDeepCopy<QString>( user_ );
    m_password = QDeepCopy<QString>( pword_ );
}

void Z3950Connection::run()
{
//  kdDebug() << "Z3950Connection::run() - " << m_fetcher->source() << endl;
    m_aborted = false;
    m_hasMore = false;
    resultsLeft = 0;
#ifdef HAVE_YAZ

    if ( !makeConnection() )
    {
        done();
        return;
    }

    ZOOM_query query = ZOOM_query_create();
    kdDebug() << "Z3950Connection::run() - pqn = " << toCString( m_pqn ) << endl;
    int errcode = ZOOM_query_prefix( query, toCString( m_pqn ) );
    if ( errcode != 0 )
    {
        kdDebug() << "Z3950Connection::run() - query error: " << m_pqn << endl;
        ZOOM_query_destroy( query );
        QString s = i18n( "Query error!" );
        s += ' ' + m_pqn;
        done( s, MessageHandler::Error );
        return;
    }

    ZOOM_resultset resultSet = ZOOM_connection_search( d->conn, query );

    // check abort status
    if ( m_aborted )
    {
        done();
        return;
    }

    // I know the LOC wants the syntax = "xml" and esn = "mods"
    // to get MODS data, that seems a bit odd...
    // esn only makes sense for marc and grs-1
    // if syntax is mods, set esn to mods too
    QCString type = "raw";
    if ( m_syntax == Latin1Literal( "mods" ) )
    {
        m_syntax = QString::fromLatin1( "xml" );
        ZOOM_resultset_option_set( resultSet, "elementSetName", "mods" );
        type = "xml";
    }
    else
    {
        ZOOM_resultset_option_set( resultSet, "elementSetName", m_esn.latin1() );
    }
    ZOOM_resultset_option_set( resultSet, "start", QCString().setNum( m_start ) );
    ZOOM_resultset_option_set( resultSet, "count", QCString().setNum( m_limit - m_start ) );
    // search in default syntax, unless syntax is already set
    if ( !m_syntax.isEmpty() )
    {
        ZOOM_resultset_option_set( resultSet, "preferredRecordSyntax", m_syntax.latin1() );
    }

    const char* errmsg;
    const char* addinfo;
    errcode = ZOOM_connection_error( d->conn, &errmsg, &addinfo );
    if ( errcode != 0 )
    {
        ZOOM_resultset_destroy( resultSet );
        ZOOM_query_destroy( query );
        m_connected = false;

        QString s = i18n( "Connection search error %1: %2" ).arg( errcode ).arg( toString( errmsg ) );
        if ( !QCString( addinfo ).isEmpty() )
        {
            s += " (" + toString( addinfo ) + ")";
        }
        kdDebug() << "Z3950Connection::run() - " << s << endl;
        done( s, MessageHandler::Error );
        return;
    }

    const size_t numResults = ZOOM_resultset_size( resultSet );

    QString newSyntax = m_syntax;
    if ( numResults > 0 )
    {
        kdDebug() << "Z3950Connection::run() - current syntax is " << m_syntax << " (" << numResults << " results)" << endl;
        // so now we know that results exist, might have to check syntax
        int len;
        ZOOM_record rec = ZOOM_resultset_record( resultSet, 0 );
        // want raw unless it's mods
        ZOOM_record_get( rec, type, &len );
        if ( len > 0 && m_syntax.isEmpty() )
        {
            newSyntax = QString::fromLatin1( ZOOM_record_get( rec, "syntax", &len ) ).lower();
            kdDebug() << "Z3950Connection::run() - syntax guess is " << newSyntax << endl;
            if ( newSyntax == Latin1Literal( "mods" ) || newSyntax == Latin1Literal( "xml" ) )
            {
                m_syntax = QString::fromLatin1( "xml" );
                ZOOM_resultset_option_set( resultSet, "elementSetName", "mods" );
            }
            else if ( newSyntax == Latin1Literal( "grs-1" ) )
            {
                // if it's defaulting to grs-1, go ahead and change it to try to get a marc
                // record since grs-1 is a last resort for us
                newSyntax.truncate( 0 );
            }
        }
        // right now, we just understand mods, unimarc, marc21/usmarc, and grs-1
        if ( newSyntax != Latin1Literal( "xml" ) &&
                newSyntax != Latin1Literal( "usmarc" ) &&
                newSyntax != Latin1Literal( "marc21" ) &&
                newSyntax != Latin1Literal( "unimarc" ) &&
                newSyntax != Latin1Literal( "grs-1" ) )
        {
            kdDebug() << "Z3950Connection::run() - changing z39.50 syntax to MODS" << endl;
            newSyntax = QString::fromLatin1( "xml" );
            ZOOM_resultset_option_set( resultSet, "elementSetName", "mods" );
            ZOOM_resultset_option_set( resultSet, "preferredRecordSyntax", newSyntax.latin1() );
            rec = ZOOM_resultset_record( resultSet, 0 );
            ZOOM_record_get( rec, "xml", &len );
            if ( len == 0 )
            {
                // change set name back
                ZOOM_resultset_option_set( resultSet, "elementSetName", m_esn.latin1() );
                newSyntax = QString::fromLatin1( "usmarc" ); // try usmarc
                kdDebug() << "Z3950Connection::run() - changing z39.50 syntax to USMARC" << endl;
                ZOOM_resultset_option_set( resultSet, "preferredRecordSyntax", newSyntax.latin1() );
                rec = ZOOM_resultset_record( resultSet, 0 );
                ZOOM_record_get( rec, "raw", &len );
            }
            if ( len == 0 )
            {
                newSyntax = QString::fromLatin1( "marc21" ); // try marc21
                kdDebug() << "Z3950Connection::run() - changing z39.50 syntax to MARC21" << endl;
                ZOOM_resultset_option_set( resultSet, "preferredRecordSyntax", newSyntax.latin1() );
                rec = ZOOM_resultset_record( resultSet, 0 );
                ZOOM_record_get( rec, "raw", &len );
            }
            if ( len == 0 )
            {
                newSyntax = QString::fromLatin1( "unimarc" ); // try unimarc
                kdDebug() << "Z3950Connection::run() - changing z39.50 syntax to UNIMARC" << endl;
                ZOOM_resultset_option_set( resultSet, "preferredRecordSyntax", newSyntax.latin1() );
                rec = ZOOM_resultset_record( resultSet, 0 );
                ZOOM_record_get( rec, "raw", &len );
            }
            if ( len == 0 )
            {
                newSyntax = QString::fromLatin1( "grs-1" ); // try grs-1
                kdDebug() << "Z3950Connection::run() - changing z39.50 syntax to GRS-1" << endl;
                ZOOM_resultset_option_set( resultSet, "preferredRecordSyntax", newSyntax.latin1() );
                rec = ZOOM_resultset_record( resultSet, 0 );
                ZOOM_record_get( rec, "raw", &len );
            }
            if ( len == 0 )
            {
                kdDebug() << "Z3950Connection::run() - giving up" << endl;
                ZOOM_resultset_destroy( resultSet );
                ZOOM_query_destroy( query );
                done( i18n( "Record syntax error" ), MessageHandler::Error );
                return;
            }
            kdDebug() << "Z3950Connection::run() - final syntax is " << newSyntax << endl;
        }
    }

    // go back to fooling ourselves and calling it mods
    if ( m_syntax == Latin1Literal( "xml" ) )
    {
        m_syntax = QString::fromLatin1( "mods" );
    }
    if ( newSyntax == Latin1Literal( "xml" ) )
    {
        newSyntax = QString::fromLatin1( "mods" );
    }
    // save syntax change for next time
    if ( m_syntax != newSyntax )
    {
        qApp->postEvent( m_fetcher, new Z3950SyntaxChange( newSyntax ) );
        m_syntax = newSyntax;
    }

    if ( m_sourceCharSet.isEmpty() )
    {
        m_sourceCharSet = QString::fromLatin1( "marc-8" );
    }

    const size_t realLimit = QMIN( numResults, m_limit );

    for ( size_t i = m_start; i < realLimit && !m_aborted; ++i )
    {
        kdDebug() << "Z3950Connection::run() - grabbing index " << i << endl;
        ZOOM_record rec = ZOOM_resultset_record( resultSet, i );
        if ( !rec )
        {
            kdDebug() << "Z3950Connection::run() - no record returned for index " << i << endl;
            continue;
        }
        int len;
        QString data;
        if ( m_syntax == Latin1Literal( "mods" ) )
        {
            data = toString( ZOOM_record_get( rec, "xml", &len ) );
        }
        else if ( m_syntax == Latin1Literal( "grs-1" ) )   // grs-1
        {
            // we're going to parse the rendered data, very ugly...
            data = toString( ZOOM_record_get( rec, "render", &len ) );
        }
        else
        {
#if 0
            kdWarning() << "Remove debug from z3950connection.cpp" << endl;
            {
                QFile f1( QString::fromLatin1( "/tmp/z3950.raw" ) );
                if ( f1.open( IO_WriteOnly ) )
                {
                    QDataStream t( &f1 );
                    t << ZOOM_record_get( rec, "raw", &len );
                }
                f1.close();
            }
#endif
            data = toXML( ZOOM_record_get( rec, "raw", &len ), m_sourceCharSet );
        }
        Z3950ResultFound* ev = new Z3950ResultFound( data );
        QApplication::postEvent( m_fetcher, ev );
    }

    ZOOM_resultset_destroy( resultSet );
    ZOOM_query_destroy( query );

    m_hasMore = m_limit < numResults;
    if ( m_hasMore )
    {
        m_start = m_limit;
        m_limit += Z3950_DEFAULT_MAX_RECORDS;
    }
#endif
    done();
}

bool Z3950Connection::makeConnection()
{
    if ( m_connected )
    {
        return true;
    }
//  kdDebug() << "Z3950Connection::makeConnection() - " << m_fetcher->source() << endl;
// I don't know what to do except assume database, user, and password are in locale encoding
#ifdef HAVE_YAZ
    d->conn_opt = ZOOM_options_create();
    ZOOM_options_set( d->conn_opt, "implementationName", "KBibTeX" );
    ZOOM_options_set( d->conn_opt, "databaseName",       toCString( m_dbname ) );
    ZOOM_options_set( d->conn_opt, "user",               toCString( m_user ) );
    ZOOM_options_set( d->conn_opt, "password",           toCString( m_password ) );

    d->conn = ZOOM_connection_create( d->conn_opt );
    ZOOM_connection_connect( d->conn, m_host.latin1(), m_port );

    int errcode;
    const char* errmsg; // unused: carries same info as 'errcode'
    const char* addinfo;
    errcode = ZOOM_connection_error( d->conn, &errmsg, &addinfo );
    if ( errcode != 0 )
    {
        ZOOM_options_destroy( d->conn_opt );
        ZOOM_connection_destroy( d->conn );
        m_connected = false;

        QString s = i18n( "Connection error %1: %2" ).arg( errcode ).arg( toString( errmsg ) );
        if ( !QCString( addinfo ).isEmpty() )
        {
            s += " (" + toString( addinfo ) + ")";
        }
        kdDebug() << "Z3950Connection::makeConnection() - " << s << endl;
        done( s, MessageHandler::Error );
        return false;
    }
#endif
    m_connected = true;
    return true;
}

void Z3950Connection::done()
{
    checkPendingEvents();
    qApp->postEvent( m_fetcher, new Z3950ConnectionDone( m_hasMore ) );
}

void Z3950Connection::done( const QString& msg_, int type_ )
{
    checkPendingEvents();
    if ( m_aborted )
    {
        qApp->postEvent( m_fetcher, new Z3950ConnectionDone( m_hasMore ) );
    }
    else
    {
        qApp->postEvent( m_fetcher, new Z3950ConnectionDone( m_hasMore, msg_, type_ ) );
    }
}

void Z3950Connection::checkPendingEvents()
{
    // if there's still some pending result events, go ahead and just wait 1 second
    if ( resultsLeft > 0 )
    {
        sleep( 1 );
    }
}

inline
QCString Z3950Connection::toCString( const QString& text_ )
{
    return iconvRun( text_.utf8(), QString::fromLatin1( "utf-8" ), m_sourceCharSet );
}

inline
QString Z3950Connection::toString( const QCString& text_ )
{
    return QString::fromUtf8( iconvRun( text_, m_sourceCharSet, QString::fromLatin1( "utf-8" ) ) );
}

// static
QCString Z3950Connection::iconvRun( const QCString& text_, const QString& fromCharSet_, const QString& toCharSet_ )
{
#ifdef HAVE_YAZ
    if ( text_.isEmpty() )
    {
        return text_;
    }

    if ( fromCharSet_ == toCharSet_ )
    {
        return text_;
    }

    yaz_iconv_t cd = yaz_iconv_open( toCharSet_.latin1(), fromCharSet_.latin1() );
    if ( !cd )
    {
        // maybe it's iso 5426, which we sorta support
        QString charSetLower = fromCharSet_.lower();
        charSetLower.remove( '-' ).remove( ' ' );
        if ( charSetLower == Latin1Literal( "iso5426" ) )
        {
            return iconvRun( Iso5426Converter::toUtf8( text_ ).utf8(), QString::fromLatin1( "utf-8" ), toCharSet_ );
        }
        else if ( charSetLower == Latin1Literal( "iso6937" ) )
        {
            return iconvRun( Iso6937Converter::toUtf8( text_ ).utf8(), QString::fromLatin1( "utf-8" ), toCharSet_ );
        }
        kdWarning() << "Z3950Connection::iconvRun() - conversion from " << fromCharSet_
        << " to " << toCharSet_ << " is unsupported" << endl;
        return text_;
    }

    const char* input = text_;
    size_t inlen = text_.length();

    size_t outlen = 2 * inlen;  // this is enough, right?
    QMemArray<char> result0( outlen );
    char* result = result0.data();

    int r = yaz_iconv( cd, const_cast<char**>( &input ), &inlen, &result, &outlen );
    if ( r <= 0 )
    {
        kdDebug() << "Z3950Connection::iconvRun() - can't decode buffer" << endl;
        return text_;
    }
    // bug in yaz, need to flush buffer to catch last character
    yaz_iconv( cd, 0, 0, &result, &outlen );

    // length is pointer difference
    size_t len = result - result0;

    QCString output = QCString( result0, len + 1 );
//  kdDebug() << "-------------------------------------------" << endl;
//  kdDebug() << output << endl;
//  kdDebug() << "-------------------------------------------" << endl;
    yaz_iconv_close( cd );
    return output;
#endif
    return text_;
}

QString Z3950Connection::toXML( const QCString& marc_, const QString& charSet_ )
{
#ifdef HAVE_YAZ
    if ( marc_.isEmpty() )
    {
        kdDebug() << "Z3950Connection::toXML() - empty string" << endl;
        return QString::null;
    }

    yaz_iconv_t cd = yaz_iconv_open( "utf-8", charSet_.latin1() );
    if ( !cd )
    {
        // maybe it's iso 5426, which we sorta support
        QString charSetLower = charSet_.lower();
        charSetLower.remove( '-' ).remove( ' ' );
        if ( charSetLower == Latin1Literal( "iso5426" ) )
        {
            return toXML( Iso5426Converter::toUtf8( marc_ ).utf8(), QString::fromLatin1( "utf-8" ) );
        }
        else if ( charSetLower == Latin1Literal( "iso6937" ) )
        {
            return toXML( Iso6937Converter::toUtf8( marc_ ).utf8(), QString::fromLatin1( "utf-8" ) );
        }
        kdWarning() << "Z3950Connection::toXML() - conversion from " << charSet_ << " is unsupported" << endl;
        return QString::null;
    }

    yaz_marc_t mt = yaz_marc_create();
    yaz_marc_iconv( mt, cd );
    yaz_marc_xml( mt, YAZ_MARC_MARCXML );

    // first 5 bytes are length
    bool ok;
#if YAZ_VERSIONL < 0x030000
    int len = marc_.left( 5 ).toInt( &ok );
#else
    size_t len = marc_.left( 5 ).toInt( &ok );
#endif
    if ( ok && ( len < 25 || len > 100000 ) )
    {
        kdDebug() << "Z3950Connection::toXML() - bad length: " << ( ok ? len : -1 ) << endl;
        return QString::null;
    }

#if YAZ_VERSIONL < 0x030000
    char* result;
#else
    const char* result;
#endif
    int r = yaz_marc_decode_buf( mt, marc_, -1, &result, &len );
    if ( r <= 0 )
    {
        kdDebug() << "Z3950Connection::toXML() - can't decode buffer" << endl;
        return QString::null;
    }

    QString output = QString::fromLatin1( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" );
    output += QString::fromUtf8( QCString( result, len + 1 ), len + 1 );
//  kdDebug() << QCString(result) << endl;
//  kdDebug() << "-------------------------------------------" << endl;
//  kdDebug() << output << endl;
    yaz_iconv_close( cd );
    yaz_marc_destroy( mt );

    return output;
#else // no yaz
    return QString::null;
#endif
}