Git commit 299a4e35adafe274ed7cec43e2c35be3f6b95ecd by Robby Stephenson. Committed on 14/06/2023 at 02:05. Pushed by rstephenson into branch 'master'.
Add initial work to fetch from OPDS catalogs Assumes the catalog has a search description url with a search template. M +4 -0 ChangeLog M +10 -0 doc/configuration.docbook M +1 -0 src/fetch/CMakeLists.txt M +2 -1 src/fetch/arxivfetcher.cpp M +2 -1 src/fetch/fetch.h M +2 -0 src/fetch/fetcherinitializer.cpp A +432 -0 src/fetch/opdsfetcher.cpp [License: GPL (v2/3)] A +123 -0 src/fetch/opdsfetcher.h [License: GPL (v2/3)] M +0 -1 src/fetch/srufetcher.cpp M +0 -1 src/fetch/z3950fetcher.cpp M +6 -0 src/tests/CMakeLists.txt A +95 -0 src/tests/opdsfetchertest.cpp [License: GPL (v2/3)] C +13 -26 src/tests/opdsfetchertest.h [from: src/translators/tellico_xml.h - 066% similarity] M +2 -0 src/translators/tellico_xml.cpp M +2 -0 src/translators/tellico_xml.h M +1 -0 xslt/CMakeLists.txt A +100 -0 xslt/atom2tellico.xsl https://invent.kde.org/office/tellico/-/commit/299a4e35adafe274ed7cec43e2c35be3f6b95ecd diff --git a/ChangeLog b/ChangeLog index 54bf361a..07a907be 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2023-06-13 Robby Stephenson <[email protected]> + + * Added OPDS catalogs as a data source. + 2023-05-29 Robby Stephenson <[email protected]> * Added support for reading images from data URLs. diff --git a/doc/configuration.docbook b/doc/configuration.docbook index cf626efd..fa8bd355 100644 --- a/doc/configuration.docbook +++ b/doc/configuration.docbook @@ -163,6 +163,7 @@ while the full list is <ulink url="https://tellico-project.org/data-sources";>ava <listitem><simpara><link linkend="amazon-web-services">Amazon.com Web Services</link>,</simpara></listitem> <listitem><simpara><link linkend="isbndb">ISBNdb.com</link>,</simpara></listitem> <listitem><simpara><link linkend="openlibrary">OpenLibrary.org</link>,</simpara></listitem> +<listitem><simpara><link linkend="opds">OPDS catalogs</link>,</simpara></listitem> <!-- movies --> <listitem><simpara>the <link linkend="imdb">Internet Movie Database</link>,</simpara></listitem> <listitem><simpara><link linkend="allocine">AlloCiné</link>,</simpara></listitem> @@ -304,6 +305,15 @@ the best known. </para> </sect3> +<sect3 id="opds"> +<title>OPDS Catalogs</title> +<para> +<ulink url="https://en.wikipedia.org/wiki/Open_Publication_Distribution_System";>OPDS catalogs</ulink> provide a means for searching (and distributing) digital books. +&tellico; can use many OPDS catalogs as a data source, such as <ulink url="https://wiki.mobileread.com/wiki/OPDS";>Project Gutenberg</ulink>. Enter the link to the catalog +and verify the access and format to confirm &tellico; can read the link. +</para> +</sect3> + </sect2> <!-- end of books --> diff --git a/src/fetch/CMakeLists.txt b/src/fetch/CMakeLists.txt index 9d7abe28..ce7b6e77 100644 --- a/src/fetch/CMakeLists.txt +++ b/src/fetch/CMakeLists.txt @@ -56,6 +56,7 @@ SET(fetch_STAT_SRCS musicbrainzfetcher.cpp numistafetcher.cpp omdbfetcher.cpp + opdsfetcher.cpp openlibraryfetcher.cpp rpggeekfetcher.cpp springerfetcher.cpp diff --git a/src/fetch/arxivfetcher.cpp b/src/fetch/arxivfetcher.cpp index 5e21a1ed..4920ee17 100644 --- a/src/fetch/arxivfetcher.cpp +++ b/src/fetch/arxivfetcher.cpp @@ -25,6 +25,7 @@ #include "arxivfetcher.h" #include "../translators/xslthandler.h" #include "../translators/tellicoimporter.h" +#include "../translators/tellico_xml.h" #include "../utils/guiproxy.h" #include "../utils/string_utils.h" #include "../utils/datafileregistry.h" @@ -164,7 +165,7 @@ void ArxivFetcher::slotComplete(KJob*) { return; } // total is top level element, with attribute totalResultsAvailable - QDomNodeList list = dom.elementsByTagNameNS(QStringLiteral("http://a9.com/-/spec/opensearch/1.1/";), + QDomNodeList list = dom.elementsByTagNameNS(XML::nsOpenSearch, QStringLiteral("totalResults")); if(list.count() > 0) { m_total = list.item(0).toElement().text().toInt(); diff --git a/src/fetch/fetch.h b/src/fetch/fetch.h index e048001c..28eed10d 100644 --- a/src/fetch/fetch.h +++ b/src/fetch/fetch.h @@ -112,7 +112,8 @@ enum Type { RPGGeek, GamingHistory, FilmAffinity, - Itunes + Itunes, + OPDS }; } diff --git a/src/fetch/fetcherinitializer.cpp b/src/fetch/fetcherinitializer.cpp index 1fc726a9..9c4a6eac 100644 --- a/src/fetch/fetcherinitializer.cpp +++ b/src/fetch/fetcherinitializer.cpp @@ -79,6 +79,7 @@ #include "gaminghistoryfetcher.h" #include "filmaffinityfetcher.h" #include "itunesfetcher.h" +#include "opdsfetcher.h" /** * Ideally, I'd like these initializations to be in each cpp file for each collection type @@ -136,6 +137,7 @@ Tellico::Fetch::FetcherInitializer::FetcherInitializer() { RegisterFetcher<Fetch::GamingHistoryFetcher> registerGamingHistory(GamingHistory); RegisterFetcher<Fetch::FilmAffinityFetcher> registerFilmAffinity(FilmAffinity); RegisterFetcher<Fetch::ItunesFetcher> registerItunes(Itunes); + RegisterFetcher<Fetch::OPDSFetcher> registerOPDS(OPDS); // these data sources depend on being able to import bibtex #ifdef ENABLE_BTPARSE diff --git a/src/fetch/opdsfetcher.cpp b/src/fetch/opdsfetcher.cpp new file mode 100644 index 00000000..b5ebd3dc --- /dev/null +++ b/src/fetch/opdsfetcher.cpp @@ -0,0 +1,432 @@ +/*************************************************************************** + Copyright (C) 2023 Robby Stephenson <[email protected]> + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License or (at your option) version 3 or any later version * + * accepted by the membership of KDE e.V. (or its successor approved * + * by the membership of KDE e.V.), which shall act as a proxy * + * defined in Section 14 of version 3 of the license. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see <http://www.gnu.org/licenses/>. * + * * + ***************************************************************************/ + +#include "opdsfetcher.h" +#include "../fieldformat.h" +#include "../collection.h" +#include "../translators/xslthandler.h" +#include "../translators/tellicoimporter.h" +#include "../gui/lineedit.h" +#include "../core/filehandler.h" +#include "../utils/datafileregistry.h" +#include "../utils/guiproxy.h" +#include "../utils/isbnvalidator.h" +#include "../translators/tellico_xml.h" +#include "../tellico_debug.h" + +#include <KLocalizedString> +#include <KIO/Job> +#include <KJobUiDelegate> +#include <KJobWidgets/KJobWidgets> +#include <KAcceleratorManager> + +#include <QLabel> +#include <QGridLayout> +#include <QXmlStreamReader> +#include <QPushButton> + +using namespace Tellico; +using Tellico::Fetch::OPDSFetcher; + +// utility class for reading the OPDS catalog and finding the search information +class OPDSReader { +public: + OPDSReader(const QUrl& catalog_) : catalog(catalog_) {} + + // read the catalog file and return the search description url + QString readSearchUrl() { + const QByteArray opdsText = FileHandler::readDataFile(catalog); + QXmlStreamReader xml(opdsText); + int depth = 0; + while(xml.readNext() != QXmlStreamReader::Invalid) { + switch(xml.tokenType()) { + case QXmlStreamReader::StartElement: + ++depth; + if(depth == 2 && + xml.name() == QLatin1String("link") && + xml.namespaceUri() == Tellico::XML::nsAtom) { + auto attributes = xml.attributes(); + if(attributes.value(QStringLiteral("rel")) == QLatin1String("search")) { + // found the search url + return attributes.value(QStringLiteral("href")).toString(); + } + } + break; + case QXmlStreamReader::EndElement: + --depth; + break; + default: + break; + } + } + // nothing found + return QString(); + } + + bool readSearchTemplate() { +// myDebug() << "Reading catalog:" << catalog; + QString searchDescriptionUrl = readSearchUrl(); + if(searchDescriptionUrl.isEmpty()) return false; +// myDebug() << "Reading search description:" << searchDescriptionUrl; + // read the search description and find the search template + const QByteArray descText = FileHandler::readDataFile(QUrl(searchDescriptionUrl)); + QXmlStreamReader xml(descText); + int depth = 0; + QString text, shortName, longName; + while(xml.readNext() != QXmlStreamReader::Invalid) { + switch(xml.tokenType()) { + case QXmlStreamReader::StartElement: + ++depth; + if(depth == 2) { + if(xml.name() == QLatin1String("Url") && + xml.namespaceUri() == XML::nsOpenSearch) { + auto attributes = xml.attributes(); + if(attributes.value(QLatin1String("type")) == QLatin1String("application/atom+xml")) { + searchTemplate = attributes.value(QStringLiteral("template")).toString(); + } + } + } + break; + case QXmlStreamReader::EndElement: + if(depth == 2 && xml.name() == QLatin1String("LongName")) { + longName = text.simplified(); + } else if(depth == 2 && xml.name() == QLatin1String("ShortName")) { + shortName = text.simplified(); + } else if(depth == 2 && xml.name() == QLatin1String("Image")) { + icon = text.simplified(); + } else if(depth == 2 && xml.name() == QLatin1String("Attribution")) { + attribution = text.simplified(); + } + --depth; + text.clear(); + break; + case QXmlStreamReader::Characters: + text += xml.text(); + break; + default: + break; + } + } + name = longName.isEmpty() ? shortName : longName; + return !searchTemplate.isEmpty(); + } + + QUrl catalog; + QString searchTemplate; + QString name; + QString icon; + QString attribution; +}; + +OPDSFetcher::OPDSFetcher(QObject* parent_) + : Fetcher(parent_), m_xsltHandler(nullptr), m_started(false) { +} + +OPDSFetcher::~OPDSFetcher() { + delete m_xsltHandler; + m_xsltHandler = nullptr; +} + +QString OPDSFetcher::source() const { + return m_name.isEmpty() ? defaultName() : m_name; +} + +QString OPDSFetcher::attribution() const { + return m_attribution; +} + +QString OPDSFetcher::icon() const { + return favIcon(QUrl(m_icon)); +} + +bool OPDSFetcher::canSearch(Fetch::FetchKey k) const { + return k == Title || k == Keyword || k == ISBN; +} + +bool OPDSFetcher::canFetch(int type) const { + return type == Data::Collection::Book || type == Data::Collection::Bibtex; +} + +void OPDSFetcher::readConfigHook(const KConfigGroup& config_) { + m_catalog = config_.readEntry("Catalog"); + m_searchTemplate = config_.readEntry("SearchTemplate"); + m_icon = config_.readEntry("Icon"); + m_attribution = config_.readEntry("Attribution"); +} + +void OPDSFetcher::saveConfigHook(KConfigGroup& config_) { + if(!m_searchTemplate.isEmpty()) { + config_.writeEntry("SearchTemplate", m_searchTemplate); + } + if(!m_icon.isEmpty()) { + config_.writeEntry("Icon", m_icon); + } + if(!m_attribution.isEmpty()) { + config_.writeEntry("Attribution", m_attribution); + } +} + +void OPDSFetcher::search() { + m_started = true; + if(m_catalog.isEmpty()) { + myDebug() << source() << "- url is not set"; + stop(); + return; + } + + OPDSReader reader(QUrl::fromUserInput(m_catalog)); + if(m_searchTemplate.isEmpty() && !reader.readSearchTemplate()) { + myDebug() << source() << "- no search template"; + message(i18n("Tellico is unable to read the search descripion in the OPDS catalog."), MessageHandler::Error); + stop(); + return; + } + if(m_searchTemplate.isEmpty()) { + m_searchTemplate = reader.searchTemplate; + m_icon = reader.icon; + m_attribution = reader.attribution; + } + + QString searchTerm; + switch(request().key()) { + case Title: + case Keyword: + searchTerm = request().value(); + break; + + case ISBN: + { + QString isbn = request().value().section(QLatin1Char(';'), 0); + isbn.remove(QLatin1Char('-')); + searchTerm = isbn; + } + break; + + default: + myWarning() << "key not recognized: " << request().key(); + stop(); + break; + } + + QString searchUrl = m_searchTemplate; + searchUrl.replace(QStringLiteral("{searchTerms}"), searchTerm); + QUrl u(searchUrl); +// myDebug() << u.url(); + + m_job = KIO::storedGet(u, KIO::NoReload, KIO::HideProgressInfo); + KJobWidgets::setWindow(m_job, GUI::Proxy::widget()); + connect(m_job.data(), &KJob::result, + this, &OPDSFetcher::slotComplete); +} + +void OPDSFetcher::stop() { + if(!m_started) { + return; + } + if(m_job) { + m_job->kill(); + m_job = nullptr; + } + + m_started = false; + emit signalDone(this); +} + +void OPDSFetcher::slotComplete(KJob*) { + if(m_job->error()) { + m_job->uiDelegate()->showErrorMessage(); + stop(); + return; + } + + QByteArray data = m_job->data(); + if(data.isEmpty()) { + stop(); + return; + } + // see bug 319662. If fetcher is cancelled, job is killed + // if the pointer is retained, it gets double-deleted + m_job = nullptr; + +#if 0 + myWarning() << "Remove debug from opdsfetcher.cpp"; + QFile f(QString::fromLatin1("/tmp/test.xml")); + if(f.open(QIODevice::WriteOnly)) { + QTextStream t(&f); + t.setCodec("UTF-8"); + t << data; + } + f.close(); +#endif + + if(!m_xsltHandler) { + initXSLTHandler(); + if(!m_xsltHandler) { // probably an error somewhere in the stylesheet loading + stop(); + return; + } + } + + // assume result is always utf-8 + QString str = m_xsltHandler->applyStylesheet(QString::fromUtf8(data.constData(), data.size())); + Import::TellicoImporter imp(str); + Data::CollPtr coll = imp.collection(); + + if(!coll) { + myDebug() << source() << " - no collection pointer"; + stop(); + return; + } + + foreach(Data::EntryPtr entry, coll->entries()) { + FetchResult* r = new FetchResult(this, entry); + m_entries.insert(r->uid, entry); + emit signalResultFound(r); + } + stop(); +} + +Tellico::Data::EntryPtr OPDSFetcher::fetchEntryHook(uint uid_) { + return m_entries[uid_]; +} + + +void OPDSFetcher::initXSLTHandler() { + QString xsltfile = DataFileRegistry::self()->locate(QStringLiteral("atom2tellico.xsl")); + if(xsltfile.isEmpty()) { + myWarning() << "can not locate atom2tellico.xsl."; + return; + } + + QUrl u = QUrl::fromLocalFile(xsltfile); + + delete m_xsltHandler; + m_xsltHandler = new XSLTHandler(u); + if(!m_xsltHandler->isValid()) { + myWarning() << "error in atom2tellico.xsl."; + delete m_xsltHandler; + m_xsltHandler = nullptr; + } +} + +Tellico::Fetch::FetchRequest OPDSFetcher::updateRequest(Data::EntryPtr entry_) { + QString t = entry_->field(QStringLiteral("title")); + if(!t.isEmpty()) { + return FetchRequest(Fetch::Title, t); + } + return FetchRequest(); +} + +QString OPDSFetcher::defaultName() { + return i18n("OPDS Catalog"); +} + +QString OPDSFetcher::defaultIcon() { + return QStringLiteral("folder-book"); +} + +// static +Tellico::StringHash OPDSFetcher::allOptionalFields() { + StringHash hash; + hash[QStringLiteral("url")] = i18n("URL"); + return hash; +} + +Tellico::Fetch::ConfigWidget* OPDSFetcher::configWidget(QWidget* parent_) const { + return new ConfigWidget(parent_, this); +} + +OPDSFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const OPDSFetcher* fetcher_ /*=0*/) + : Fetch::ConfigWidget(parent_) { + QGridLayout* l = new QGridLayout(optionsWidget()); + l->setSpacing(4); + l->setColumnStretch(1, 10); + + int row = -1; + QLabel* label = new QLabel(i18n("Catalog: "), optionsWidget()); + l->addWidget(label, ++row, 0); + m_catalogEdit = new GUI::LineEdit(optionsWidget()); + connect(m_catalogEdit, &QLineEdit::textChanged, this, &ConfigWidget::slotSetModified); + l->addWidget(m_catalogEdit, row, 1); + QString w = i18n("Enter the link to the OPDS server."); + label->setWhatsThis(w); + m_catalogEdit->setWhatsThis(w); + label->setBuddy(m_catalogEdit); + + auto verifyButton = new QPushButton(i18n("&Verify Catalog"), optionsWidget()); + connect(verifyButton, &QPushButton::clicked, + this, &ConfigWidget::verifyCatalog); + l->addWidget(verifyButton, ++row, 0); + m_statusLabel = new QLabel(optionsWidget()); + l->addWidget(m_statusLabel, row, 1); + + l->setRowStretch(++row, 1); + + // now add additional fields widget + addFieldsWidget(OPDSFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList()); + + if(fetcher_) { + m_catalogEdit->setText(fetcher_->m_catalog); + m_searchTemplate = fetcher_->m_searchTemplate; + m_icon = fetcher_->m_icon; + m_attribution = fetcher_->m_attribution; + } + KAcceleratorManager::manage(optionsWidget()); +} + +void OPDSFetcher::ConfigWidget::saveConfigHook(KConfigGroup& config_) { + QString s = m_catalogEdit->text().trimmed(); + if(!s.isEmpty()) { + config_.writeEntry("Catalog", s); + config_.writeEntry("SearchTemplate", m_searchTemplate); + config_.writeEntry("Icon", m_icon); + config_.writeEntry("Attribution", m_attribution); + } +} + +QString OPDSFetcher::ConfigWidget::preferredName() const { + QString s = m_catalogEdit->text(); + return s.isEmpty() ? OPDSFetcher::defaultName() : s; +} + +void OPDSFetcher::ConfigWidget::verifyCatalog() { + OPDSReader reader(QUrl::fromUserInput(m_catalogEdit->text())); + if(reader.readSearchTemplate()) { + const int imgSize = 0.8*m_statusLabel->height(); + m_statusLabel->setPixmap(QIcon::fromTheme(QStringLiteral("emblem-checked")).pixmap(imgSize, imgSize)); + slotSetModified(); + if(!reader.name.isEmpty()) { + emit signalName(reader.name); + } + m_searchTemplate = reader.searchTemplate; + m_icon = reader.icon; + m_attribution = reader.attribution; + } else { + const int imgSize = 0.8*m_statusLabel->height(); + m_statusLabel->setPixmap(QIcon::fromTheme(QStringLiteral("emblem-error")).pixmap(imgSize, imgSize)); + m_searchTemplate.clear(); + m_icon.clear(); + m_attribution.clear(); + } +} diff --git a/src/fetch/opdsfetcher.h b/src/fetch/opdsfetcher.h new file mode 100644 index 00000000..a315e1bb --- /dev/null +++ b/src/fetch/opdsfetcher.h @@ -0,0 +1,123 @@ +/*************************************************************************** + Copyright (C) 2023 Robby Stephenson <[email protected]> + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License or (at your option) version 3 or any later version * + * accepted by the membership of KDE e.V. (or its successor approved * + * by the membership of KDE e.V.), which shall act as a proxy * + * defined in Section 14 of version 3 of the license. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see <http://www.gnu.org/licenses/>. * + * * + ***************************************************************************/ + +#ifndef TELLICO_OPDSFETCHER_H +#define TELLICO_OPDSFETCHER_H + +#include "fetcher.h" +#include "configwidget.h" + +#include <QPointer> + +class QLabel; +class KJob; +namespace KIO { + class StoredTransferJob; +} + +namespace Tellico { + class XSLTHandler; + namespace GUI { + class LineEdit; + } + namespace Fetch { + +/** + * @author Robby Stephenson + */ +class OPDSFetcher : public Fetcher { +Q_OBJECT + +public: + /** + */ + OPDSFetcher(QObject* parent); + /** + */ + virtual ~OPDSFetcher(); + + /** + */ + virtual QString source() const Q_DECL_OVERRIDE; + virtual QString attribution() const Q_DECL_OVERRIDE; + virtual QString icon() const Q_DECL_OVERRIDE; + virtual bool isSearching() const Q_DECL_OVERRIDE { return m_started; } + virtual bool canSearch(FetchKey k) const Q_DECL_OVERRIDE; + virtual void stop() Q_DECL_OVERRIDE; + virtual Data::EntryPtr fetchEntryHook(uint uid) Q_DECL_OVERRIDE; + virtual Type type() const Q_DECL_OVERRIDE { return OPDS; } + virtual bool canFetch(int type) const Q_DECL_OVERRIDE; + virtual void readConfigHook(const KConfigGroup& config) Q_DECL_OVERRIDE; + virtual void saveConfigHook(KConfigGroup& config) Q_DECL_OVERRIDE; + + virtual Fetch::ConfigWidget* configWidget(QWidget* parent) const Q_DECL_OVERRIDE; + + class ConfigWidget; + friend class ConfigWidget; + + static QString defaultName(); + static QString defaultIcon(); + static StringHash allOptionalFields(); + +private Q_SLOTS: + void slotComplete(KJob* job); + +private: + virtual void search() Q_DECL_OVERRIDE; + virtual FetchRequest updateRequest(Data::EntryPtr entry) Q_DECL_OVERRIDE; + void initXSLTHandler(); + + QString m_catalog; + QString m_searchTemplate; + QString m_icon; + QString m_attribution; + XSLTHandler* m_xsltHandler; + + QHash<uint, Data::EntryPtr> m_entries; + QPointer<KIO::StoredTransferJob> m_job; + bool m_started; +}; + +class OPDSFetcher::ConfigWidget : public Fetch::ConfigWidget { +Q_OBJECT + +public: + explicit ConfigWidget(QWidget* parent_, const OPDSFetcher* fetcher = nullptr); + virtual void saveConfigHook(KConfigGroup& config) Q_DECL_OVERRIDE; + virtual QString preferredName() const Q_DECL_OVERRIDE; + +private Q_SLOTS: + void verifyCatalog(); + +private: + GUI::LineEdit* m_catalogEdit; + QLabel* m_statusLabel; + QString m_searchTemplate; + QString m_icon; + QString m_attribution; +}; + + } // end namespace +} // end namespace +#endif diff --git a/src/fetch/srufetcher.cpp b/src/fetch/srufetcher.cpp index 5f5fe5e7..d1ba258e 100644 --- a/src/fetch/srufetcher.cpp +++ b/src/fetch/srufetcher.cpp @@ -498,7 +498,6 @@ QString SRUFetcher::defaultName() { } QString SRUFetcher::defaultIcon() { -// return QLatin1String("network-workgroup"); // just to be different than z3950 return QStringLiteral(":/icons/sru"); } diff --git a/src/fetch/z3950fetcher.cpp b/src/fetch/z3950fetcher.cpp index e936eacc..6df61dca 100644 --- a/src/fetch/z3950fetcher.cpp +++ b/src/fetch/z3950fetcher.cpp @@ -567,7 +567,6 @@ QString Z3950Fetcher::defaultName() { } QString Z3950Fetcher::defaultIcon() { -// return QLatin1String("network-server"); // rather arbitrary return QStringLiteral("network-server-database"); // rather arbitrary } diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index b77f0846..22bb66da 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -773,6 +773,12 @@ ecm_add_test(numistafetchertest.cpp LINK_LIBRARIES fetcherstest ${TELLICO_TEST_LIBS} ) +ecm_add_test(opdsfetchertest.cpp + ../fetch/opdsfetcher.cpp + TEST_NAME opdsfetchertest + LINK_LIBRARIES fetcherstest ${TELLICO_TEST_LIBS} +) + ecm_add_test(openlibraryfetchertest.cpp ../fetch/openlibraryfetcher.cpp TEST_NAME openlibraryfetchertest diff --git a/src/tests/opdsfetchertest.cpp b/src/tests/opdsfetchertest.cpp new file mode 100644 index 00000000..0195232a --- /dev/null +++ b/src/tests/opdsfetchertest.cpp @@ -0,0 +1,95 @@ +/*************************************************************************** + Copyright (C) 2023 Robby Stephenson <[email protected]> + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License or (at your option) version 3 or any later version * + * accepted by the membership of KDE e.V. (or its successor approved * + * by the membership of KDE e.V.), which shall act as a proxy * + * defined in Section 14 of version 3 of the license. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see <http://www.gnu.org/licenses/>. * + * * + ***************************************************************************/ + +#undef QT_NO_CAST_FROM_ASCII + +#include "opdsfetchertest.h" + +#include "../fetch/opdsfetcher.h" +#include "../collections/bookcollection.h" +#include "../collectionfactory.h" +#include "../entry.h" +#include "../images/imagefactory.h" +#include "../utils/datafileregistry.h" + +#include <KSharedConfig> +#include <KConfigGroup> + +#include <QTest> + +QTEST_GUILESS_MAIN( OPDSFetcherTest ) + +OPDSFetcherTest::OPDSFetcherTest() : AbstractFetcherTest() { + QStandardPaths::setTestModeEnabled(true); +} + +void OPDSFetcherTest::initTestCase() { + Tellico::RegisterCollection<Tellico::Data::BookCollection> registerBook(Tellico::Data::Collection::Book, "book"); + Tellico::DataFileRegistry::self()->addDataLocation(QFINDTESTDATA("../../xslt/atom2tellico.xsl")); + Tellico::ImageFactory::init(); +} + +void OPDSFetcherTest::testFeedbooksSearch() { + KConfigGroup cg = KSharedConfig::openConfig(QString(), KConfig::SimpleConfig)->group("Feedbooks"); + cg.writeEntry("Catalog", "https://www.feedbooks.com/catalog.atom";); + cg.writeEntry("Custom Fields", "url"); + + Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Book, Tellico::Fetch::ISBN, + "9781773231341"); + Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::OPDSFetcher(this)); + fetcher->readConfig(cg); + + Tellico::Data::EntryList results = DO_FETCH(fetcher, request); + + QCOMPARE(results.size(), 1); + + Tellico::Data::EntryPtr entry = results.at(0); + QCOMPARE(entry->field("title"), "First Lensman"); + QCOMPARE(entry->field("author"), "E. E. Smith"); + QCOMPARE(entry->field("isbn"), "978-1-77323-134-1"); + QCOMPARE(entry->field("pub_year"), "2018"); + QCOMPARE(entry->field("publisher"), "Reading Essentials"); + QCOMPARE(entry->field("genre"), "Fiction; Science fiction; Space opera and planet opera"); + QCOMPARE(entry->field("pages"), "226"); + QCOMPARE(entry->field("url"), "https://www.feedbooks.com/item/2971293";); + QVERIFY(!entry->field("cover").isEmpty()); + QVERIFY(!entry->field("cover").contains(QLatin1Char('/'))); + QVERIFY(!entry->field("plot").isEmpty()); +} + +void OPDSFetcherTest::testEmptyGutenberg() { + KConfigGroup cg = KSharedConfig::openConfig(QString(), KConfig::SimpleConfig)->group("Feedbooks"); + cg.writeEntry("Catalog", "https://m.gutenberg.org/ebooks.opds/";); + + Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Book, Tellico::Fetch::Title, + "XXXXXXXXXXXX"); + Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::OPDSFetcher(this)); + fetcher->readConfig(cg); + + Tellico::Data::EntryList results = DO_FETCH(fetcher, request); + + // should be no results + QVERIFY(results.isEmpty()); + QVERIFY(!fetcher->attribution().isEmpty()); +} diff --git a/src/translators/tellico_xml.h b/src/tests/opdsfetchertest.h similarity index 66% copy from src/translators/tellico_xml.h copy to src/tests/opdsfetchertest.h index 886f43a2..08276772 100644 --- a/src/translators/tellico_xml.h +++ b/src/tests/opdsfetchertest.h @@ -1,5 +1,5 @@ /*************************************************************************** - Copyright (C) 2003-2009 Robby Stephenson <[email protected]> + Copyright (C) 2023 Robby Stephenson <[email protected]> ***************************************************************************/ /*************************************************************************** @@ -22,33 +22,20 @@ * * ***************************************************************************/ -#ifndef TELLICO_XML_H -#define TELLICO_XML_H +#ifndef OPDSFETCHERTEST_H +#define OPDSFETCHERTEST_H -#include <QString> +#include "abstractfetchertest.h" -namespace Tellico { - namespace XML { - extern const QString nsXSL; - extern const QString nsBibtexml; - extern const QString dtdBibtexml; +class OPDSFetcherTest : public AbstractFetcherTest { +Q_OBJECT +public: + OPDSFetcherTest(); - extern const uint syntaxVersion; - extern const QString nsTellico; - - QString pubTellico(int version = syntaxVersion); - QString dtdTellico(int version = syntaxVersion); - - extern const QString nsBookcase; - extern const QString nsDublinCore; - extern const QString nsZing; - extern const QString nsZingDiag; - - bool validXMLElementName(const QString& name); - QString elementName(const QString& name); - QByteArray recoverFromBadXMLName(const QByteArray& data); - QByteArray removeInvalidXml(const QByteArray& data); - } -} +private Q_SLOTS: + void initTestCase(); + void testFeedbooksSearch(); + void testEmptyGutenberg(); +}; #endif diff --git a/src/translators/tellico_xml.cpp b/src/translators/tellico_xml.cpp index bebdefa0..a8854769 100644 --- a/src/translators/tellico_xml.cpp +++ b/src/translators/tellico_xml.cpp @@ -70,6 +70,8 @@ const QString Tellico::XML::nsBookcase = QStringLiteral("http://periapsis.org/bo const QString Tellico::XML::nsDublinCore = QStringLiteral("http://purl.org/dc/elements/1.1/";); const QString Tellico::XML::nsZing = QStringLiteral("http://www.loc.gov/zing/srw/";); const QString Tellico::XML::nsZingDiag = QStringLiteral("http://www.loc.gov/zing/srw/diagnostic/";); +const QString Tellico::XML::nsAtom = QStringLiteral("http://www.w3.org/2005/Atom";); +const QString Tellico::XML::nsOpenSearch = QStringLiteral("http://a9.com/-/spec/opensearch/1.1/";); QString Tellico::XML::pubTellico(int version) { return QStringLiteral("-//Robby Stephenson/DTD Tellico V%1.0//EN").arg(version); diff --git a/src/translators/tellico_xml.h b/src/translators/tellico_xml.h index 886f43a2..40b52cfc 100644 --- a/src/translators/tellico_xml.h +++ b/src/translators/tellico_xml.h @@ -43,6 +43,8 @@ namespace Tellico { extern const QString nsDublinCore; extern const QString nsZing; extern const QString nsZingDiag; + extern const QString nsAtom; + extern const QString nsOpenSearch; bool validXMLElementName(const QString& name); QString elementName(const QString& name); diff --git a/xslt/CMakeLists.txt b/xslt/CMakeLists.txt index edfe496e..46259e4c 100644 --- a/xslt/CMakeLists.txt +++ b/xslt/CMakeLists.txt @@ -5,6 +5,7 @@ ADD_SUBDIRECTORY( report-templates ) SET(XSLT_FILES arxiv2tellico.xsl + atom2tellico.xsl biblioshare2tellico.xsl bibtexml2tellico.xsl bluray-logo.png diff --git a/xslt/atom2tellico.xsl b/xslt/atom2tellico.xsl new file mode 100644 index 00000000..c843bead --- /dev/null +++ b/xslt/atom2tellico.xsl @@ -0,0 +1,100 @@ +<?xml version="1.0"?> +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"; + xmlns="http://periapsis.org/tellico/"; + xmlns:atom="http://www.w3.org/2005/Atom"; + xmlns:dcterms="http://purl.org/dc/terms/"; + xmlns:schema="http://schema.org"; + xmlns:str="http://exslt.org/strings"; + xmlns:exsl="http://exslt.org/common"; + exclude-result-prefixes="atom dcterms schema" + extension-element-prefixes="str exsl" + version="1.0"> + +<!-- + =================================================================== + Tellico XSLT file - used for importing data from an atom feed + + Copyright (C) 2023 Robby Stephenson <[email protected]> + + This XSLT stylesheet is designed to be used with the 'Tellico' + application, which can be found at http://tellico-project.org + + =================================================================== +--> + +<xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes" + doctype-public="-//Robby Stephenson/DTD Tellico V11.0//EN" + doctype-system="http://periapsis.org/tellico/dtd/v11/tellico.dtd"/> + +<xsl:template match="/"> + <tellico syntaxVersion="11"> + <collection title="Atom Search" type="2"> + <fields> + <field name="_default"/> + <field flags="0" title="URL" category="General" format="4" type="7" name="url" i18n="true"/> + </fields> + <!-- Project Gutenberg returns an entry without author when there are no search results --> + <xsl:apply-templates select="atom:feed/atom:entry[atom:author]"/> + </collection> + </tellico> +</xsl:template> + +<xsl:template match="atom:entry"> + <entry> + + <title> + <xsl:value-of select="normalize-space(atom:title)"/> + </title> + + <authors> + <xsl:for-each select="atom:author"> + <author> + <xsl:value-of select="normalize-space(atom:name)"/> + </author> + </xsl:for-each> + </authors> + + <publishers> + <xsl:for-each select="dcterms:publisher"> + <publisher> + <xsl:value-of select="normalize-space(.)"/> + </publisher> + </xsl:for-each> + </publishers> + + <url> + <xsl:value-of select="atom:id[starts-with(.,'http')]"/> + </url> + + <isbn> + <xsl:value-of select="substring-after(dcterms:identifier[starts-with(.,'urn:ISBN')],'ISBN:')"/> + </isbn> + + <pub_year> + <xsl:value-of select="substring(dcterms:issued,1,4)"/> + </pub_year> + + <pages> + <xsl:value-of select="schema:numberOfPages"/> + </pages> + + <plot> + <xsl:value-of select="normalize-space(atom:summary)"/> + </plot> + + <cover> + <xsl:value-of select="(atom:link[@rel='http://opds-spec.org/image']/@href | + atom:link[@rel='http://opds-spec.org/image/thumbnail']/@href)[1]"/> + </cover> + + <genres> + <xsl:for-each select="atom:category"> + <genre> + <xsl:value-of select="@label"/> + </genre> + </xsl:for-each> + </genres> + </entry> +</xsl:template> + +</xsl:stylesheet>
