Git commit a4207de7ba5d1cfb5ec7c397e2dd161e1e509fd3 by Robby Stephenson. Committed on 01/01/2020 at 01:55. Pushed by rstephenson into branch 'master'.
Add Colnect data source M +4 -0 ChangeLog M +1 -1 TODO M +18 -1 doc/configuration.docbook M +1 -0 src/fetch/CMakeLists.txt A +548 -0 src/fetch/colnectfetcher.cpp [License: GPL (v2/3)] A +132 -0 src/fetch/colnectfetcher.h [License: GPL (v2/3)] M +2 -1 src/fetch/fetch.h M +2 -0 src/fetch/fetcherinitializer.cpp M +5 -0 src/fetch/fetchresult.cpp M +8 -0 src/tests/CMakeLists.txt A +130 -0 src/tests/colnectfetchertest.cpp [License: GPL (v2/3)] C +14 -83 src/tests/colnectfetchertest.h [from: src/fetch/fetch.h - 057% similarity] M +3 -0 src/tests/tellicotest.config https://commits.kde.org/tellico/a4207de7ba5d1cfb5ec7c397e2dd161e1e509fd3 diff --git a/ChangeLog b/ChangeLog index fe2f8e5c..e2703f4a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2019-12-10 Robby Stephenson <[email protected]> + + * Added fetcher for colnect. + 2019-11-25 Robby Stephenson <[email protected]> * Fixed performance regression when loading data file. diff --git a/TODO b/TODO index 2185539b..bfd864d5 100644 --- a/TODO +++ b/TODO @@ -23,6 +23,6 @@ Replace most of the toggle actions in the view settings with KDualAction Update the video game XSLT data sources to somehow use the platform normalization -Colnect API https://colnect.com +Add stamps search from Colnect Investigate using KINO-Teatr.ua API, https://api.kino-teatr.ua Use KRatingWidget diff --git a/doc/configuration.docbook b/doc/configuration.docbook index ff57cdcf..71585153 100644 --- a/doc/configuration.docbook +++ b/doc/configuration.docbook @@ -182,11 +182,14 @@ while the full list is <ulink url="http://tellico-project.org/data-sources";>avai <listitem><simpara><link linkend="videogamegeek">VideoGameGeek</link>,</simpara></listitem> <!-- board games --> <listitem><simpara><link linkend="boardgamegeek">BoardGameGeek</link>,</simpara></listitem> -<!-- bibliographic and others --> +<!-- bibliographic --> <listitem><simpara><link linkend="bib-sources">arxiv.org</link>,</simpara></listitem> <listitem><simpara><link linkend="entrez">Entrez (PubMed) databases</link>,</simpara></listitem> <listitem><simpara><link linkend="z3950">z39.50 servers</link>,</simpara></listitem> <listitem><simpara><link linkend="sru">SRU servers</link>,</simpara></listitem> +<!-- coins --> +<listitem><simpara><link linkend="colnect">Colnect</link>,</simpara></listitem> +<!-- others --> <listitem><simpara><link linkend="externalexec">other external scripts or applications</link>, and</simpara></listitem> <listitem><simpara><link linkend="multiple-sources">combinations of any of the above sources</link>.</simpara></listitem> </itemizedlist> @@ -450,6 +453,20 @@ The <ulink url="http://www.imdb.com";>Internet Movie Database</ulink> provides in </sect2> +<!-- start of coin sources --> +<sect2 id="coin-sources"> +<title>Coin Data Sources</title> + +<sect3 id="colnect"> +<title>Colnect</title> +<para> +<ulink url="https://colnect.com";>Colnect</ulink> is an online community for collectibles providing personal collection management. +&tellico; can search Colnect for coin information. +</para> +</sect3> + +</sect2> + <sect2 id="variety-type-sources"> <title>Data Sources for Multiple Collection Types</title> diff --git a/src/fetch/CMakeLists.txt b/src/fetch/CMakeLists.txt index 6005766b..17ad8343 100644 --- a/src/fetch/CMakeLists.txt +++ b/src/fetch/CMakeLists.txt @@ -12,6 +12,7 @@ SET(fetch_STAT_SRCS bibliosharefetcher.cpp bibsonomyfetcher.cpp boardgamegeekfetcher.cpp + colnectfetcher.cpp comicvinefetcher.cpp configwidget.cpp crossreffetcher.cpp diff --git a/src/fetch/colnectfetcher.cpp b/src/fetch/colnectfetcher.cpp new file mode 100644 index 00000000..b09bc0b4 --- /dev/null +++ b/src/fetch/colnectfetcher.cpp @@ -0,0 +1,548 @@ +/*************************************************************************** + Copyright (C) 2019 Robby Stephenson <[email protected]> + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License or (at your option) version 3 or any later version * + * accepted by the membership of KDE e.V. (or its successor approved * + * by the membership of KDE e.V.), which shall act as a proxy * + * defined in Section 14 of version 3 of the license. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see <http://www.gnu.org/licenses/>. * + * * + ***************************************************************************/ + +#include "colnectfetcher.h" +#include "../collections/coincollection.h" +#include "../images/imagefactory.h" +#include "../gui/combobox.h" +#include "../utils/guiproxy.h" +#include "../utils/string_utils.h" +#include "../entry.h" +#include "../fieldformat.h" +#include "../core/filehandler.h" +#include "../tellico_debug.h" + +#include <KLocalizedString> +#include <KConfigGroup> +#include <KJob> +#include <KJobUiDelegate> +#include <KJobWidgets/KJobWidgets> +#include <KIO/StoredTransferJob> + +#include <QLabel> +#include <QFile> +#include <QTextStream> +#include <QGridLayout> +#include <QTextCodec> +#include <QJsonDocument> +#include <QJsonArray> +#include <QJsonValue> +#include <QRegularExpression> +#include <QStandardPaths> + +namespace { + static const char* COLNECT_API_URL = "https://api.tellico-project.org/colnect";; +// static const char* COLNECT_API_URL = "https://api.colnect.net";; + static const char* COLNECT_IMAGE_URL = "https://i.colnect.net";; +} + +using namespace Tellico; +using Tellico::Fetch::ColnectFetcher; + +ColnectFetcher::ColnectFetcher(QObject* parent_) + : Fetcher(parent_) + , m_started(false) + , m_locale(QStringLiteral("en")) { +} + +ColnectFetcher::~ColnectFetcher() { +} + +QString ColnectFetcher::source() const { + return m_name.isEmpty() ? defaultName() : m_name; +} + +QString ColnectFetcher::attribution() const { + return QStringLiteral("Catalog information courtesy of Colnect, an online collectors community."); +} + +bool ColnectFetcher::canSearch(FetchKey k) const { + return k == Keyword; +} + +bool ColnectFetcher::canFetch(int type) const { + return type == Data::Collection::Coin; +} + +void ColnectFetcher::readConfigHook(const KConfigGroup& config_) { + QString k = config_.readEntry("Locale", "en"); + if(!k.isEmpty()) { + m_locale = k.toLower(); + } + Q_ASSERT_X(m_locale.length() == 2, "ColnectFetcher::readConfigHook", "lang should be 2 char short iso"); +} + +void ColnectFetcher::search() { + m_started = true; + m_year.clear(); + + QUrl u(QString::fromLatin1(COLNECT_API_URL)); + // Colnect API calls are encoded as a path + QString query(QLatin1Char('/') + m_locale); + + QString value = request().value; + switch(request().key) { + case Keyword: + { + query += QStringLiteral("/list/cat/coins"); + // pull out mint year, keep the regexp a little loose + QRegularExpression yearRX(QStringLiteral("[0-9]{4}")); + QRegularExpressionMatch match = yearRX.match(value); + if(match.hasMatch()) { + m_year = match.captured(0); + query += QStringLiteral("/mint_year/") + m_year; + value = value.remove(yearRX); + } + } + // everything left is for the item description + query += QStringLiteral("/description/") + value.simplified(); + break; + + case Raw: + query += QStringLiteral("/item/cat/coins/id/") + value; + break; + + default: + myWarning() << "key not recognized:" << request().key; + stop(); + return; + } + + u.setPath(u.path() + query); +// myDebug() << "url:" << u; + + m_job = KIO::storedGet(u, KIO::NoReload, KIO::HideProgressInfo); + KJobWidgets::setWindow(m_job, GUI::Proxy::widget()); + connect(m_job.data(), &KJob::result, this, &ColnectFetcher::slotComplete); +} + +void ColnectFetcher::stop() { + if(!m_started) { + return; + } + if(m_job) { + m_job->kill(); + m_job = nullptr; + } + m_started = false; + emit signalDone(this); +} + +Tellico::Data::EntryPtr ColnectFetcher::fetchEntryHook(uint uid_) { + Data::EntryPtr entry = m_entries.value(uid_); + if(!entry) { + myWarning() << "no entry in dict"; + return Data::EntryPtr(); + } + + // if there's a colnect-id in the entry, need to fetch all the data + const QString id = entry->field(QStringLiteral("colnect-id")); + if(!id.isEmpty()) { + QUrl u(QString::fromLatin1(COLNECT_API_URL)); + QString query(QLatin1Char('/') + m_locale + QStringLiteral("/item/cat/coins/id/") + id); + u.setPath(u.path() + query); +// myDebug() << "Reading item data from url:" << u; + + QPointer<KIO::StoredTransferJob> job = KIO::storedGet(u, KIO::NoReload, KIO::HideProgressInfo); + KJobWidgets::setWindow(job, GUI::Proxy::widget()); + if(!job->exec()) { + myDebug() << "Colnect item data:" << job->errorString() << u; + return entry; + } + const QByteArray data = job->data(); + if(data.isEmpty()) { + myDebug() << "no colnect item data for" << u; + return entry; + } +#if 0 + myWarning() << "Remove item debug from colnectfetcher.cpp"; + QFile file(QStringLiteral("/tmp/colnectitemtest.json")); + if(file.open(QIODevice::WriteOnly)) { + QTextStream t(&file); + t.setCodec("UTF-8"); + t << data; + } + file.close(); +#endif + QJsonParseError jsonError; + QJsonDocument doc = QJsonDocument::fromJson(data, &jsonError); + Q_ASSERT_X(!doc.isNull(), "colnect", jsonError.errorString().toUtf8().constData()); + const QVariantList resultList = doc.array().toVariantList(); + Q_ASSERT_X(!resultList.isEmpty(), "colnect", "no item results"); + Q_ASSERT_X(static_cast<QMetaType::Type>(resultList.at(0).type()) == QMetaType::QString, "colnect", + "Weird single item result, first value is not a string"); + populateEntry(entry, resultList); + } + + // image might still be a URL only + QString image = entry->field(QStringLiteral("obverse")); + if(image.contains(QLatin1Char('/'))) { + const QString id = ImageFactory::addImage(QUrl::fromUserInput(image), true /* quiet */); + if(id.isEmpty()) { + message(i18n("The cover image could not be loaded."), MessageHandler::Warning); + } + // empty image ID is ok + entry->setField(QStringLiteral("obverse"), id); + } + // now the reverse image + image = entry->field(QStringLiteral("reverse")); + if(image.contains(QLatin1Char('/'))) { + const QString id = ImageFactory::addImage(QUrl::fromUserInput(image), true /* quiet */); + if(id.isEmpty()) { + message(i18n("The cover image could not be loaded."), MessageHandler::Warning); + } + entry->setField(QStringLiteral("reverse"), id); + } + + // don't want to include id + entry->setField(QStringLiteral("colnect-id"), QString()); + return entry; +} + +Tellico::Fetch::FetchRequest ColnectFetcher::updateRequest(Data::EntryPtr entry_) { + const QString title = entry_->field(QStringLiteral("title")); + if(!title.isEmpty()) { + return FetchRequest(Keyword, title); + } + return FetchRequest(); +} + +void ColnectFetcher::slotComplete(KJob* job_) { + KIO::StoredTransferJob* job = static_cast<KIO::StoredTransferJob*>(job_); + + if(job->error()) { + job->uiDelegate()->showErrorMessage(); + stop(); + return; + } + + const QByteArray data = job->data(); + if(data.isEmpty()) { + myDebug() << "no data"; + stop(); + return; + } + // see bug 319662. If fetcher is cancelled, job is killed + // if the pointer is retained, it gets double-deleted + m_job = nullptr; + +#if 0 + myWarning() << "Remove debug from colnectfetcher.cpp"; + QFile f(QStringLiteral("/tmp/colnecttest.json")); + if(f.open(QIODevice::WriteOnly)) { + QTextStream t(&f); + t.setCodec("UTF-8"); + t << data; + } + f.close(); +#endif + + QJsonParseError jsonError; + QJsonDocument doc = QJsonDocument::fromJson(data, &jsonError); + if(doc.isNull()) { + myDebug() << "null JSON document:" << jsonError.errorString(); + message(jsonError.errorString(), MessageHandler::Error); + stop(); + return; + } + QVariantList resultList = doc.array().toVariantList(); + if(resultList.isEmpty()) { + myDebug() << "no results"; + stop(); + return; + } + + m_hasMoreResults = false; // for now, no continued searches + + Data::CollPtr coll(new Data::CoinCollection(true)); + // placeholder for colnect id, to be removed later + Data::FieldPtr f1(new Data::Field(QStringLiteral("colnect-id"), QString())); + coll->addField(f1); + + const QString series(QStringLiteral("series")); + if(!coll->hasField(series) && optionalFields().contains(series)) { + Data::FieldPtr field(new Data::Field(series, i18n("Series"))); + field->setCategory(i18n("General")); + coll->addField(field); + } + + const QString desc(QStringLiteral("description")); + if(!coll->hasField(desc) && optionalFields().contains(desc)) { + Data::FieldPtr field(new Data::Field(desc, i18n("Description"), Data::Field::Para)); + coll->addField(field); + } + + const QString mintage(QStringLiteral("mintage")); + if(!coll->hasField(mintage) && optionalFields().contains(mintage)) { + Data::FieldPtr field(new Data::Field(mintage, i18n("Mintage"), Data::Field::Number)); + field->setCategory(i18n("General")); + coll->addField(field); + } + + // if the first item in the array is a string, probably a single item result, possibly from a Raw query + if(!resultList.isEmpty() && + static_cast<QMetaType::Type>(resultList.at(0).type()) == QMetaType::QString) { + Data::EntryPtr entry(new Data::Entry(coll)); + populateEntry(entry, resultList); + + FetchResult* r = new FetchResult(Fetcher::Ptr(this), entry); + m_entries.insert(r->uid, entry); + emit signalResultFound(r); + + stop(); + return; + } + + // here, we have multiple results to loop through +// myDebug() << "Reading" << resultList.size() << "results"; + foreach(const QVariant& result, resultList) { + // be sure to check that the fetcher has not been stopped + // crashes can occur if not + if(!m_started) { + break; + } + + Data::EntryPtr entry(new Data::Entry(coll)); + //list action - returns array of [item_id,series_id,producer_id,front_picture_id, back_picture_id,item_description,catalog_codes,item_name] + const QVariantList values = result.toJsonArray().toVariantList(); + entry->setField(QStringLiteral("colnect-id"), values.first().toString()); + entry->setField(QStringLiteral("description"), values.last().toString()); + entry->setField(QStringLiteral("year"), m_year); + + FetchResult* r = new FetchResult(Fetcher::Ptr(this), entry); + m_entries.insert(r->uid, entry); + emit signalResultFound(r); + } + + stop(); +} + +void ColnectFetcher::populateEntry(Data::EntryPtr entry_, const QVariantList& resultList_) { + if(m_colnectFields.isEmpty()) { + readDataList(); + // set minimum size of list here + if(m_colnectFields.count() < 26) { + return; + } + } + if(resultList_.count() != m_colnectFields.count()) { + myDebug() << "field count mismatch! Got" << resultList_.count() << ", expected" << m_colnectFields.count(); + return; + } + + // lookup the field name for the list index + int idx = m_colnectFields.value(QStringLiteral("Issued on"), -1); + + // the year may have already been set in the query term + if(m_year.isEmpty() && idx > -1) { + entry_->setField(QStringLiteral("year"), resultList_.at(idx).toString()); + } + + idx = m_colnectFields.value(QStringLiteral("Country"), -1); + if(idx > -1) { + entry_->setField(QStringLiteral("country"), resultList_.at(idx).toString()); + } + + idx = m_colnectFields.value(QStringLiteral("Currency"), -1); + if(idx > -1) { + entry_->setField(QStringLiteral("currency"), resultList_.at(idx).toString()); + idx = m_colnectFields.value(QStringLiteral("FaceValue"), -1); + if(idx > -1) { + // bad assumption, but go with it. First char is currency symbol + QString currency = entry_->field(QStringLiteral("currency")); + if(!currency.isEmpty()) currency.truncate(1); + const double value = resultList_.at(idx).toDouble(); + entry_->setField(QStringLiteral("denomination"), + QLocale::system().toCurrencyString(value, currency)); + } + } + + idx = m_colnectFields.value(QStringLiteral("Series"), -1); + static const QString series(QStringLiteral("series")); + if(idx > -1 && optionalFields().contains(series)) { + entry_->setField(series, resultList_.at(idx).toString()); + } + + idx = m_colnectFields.value(QStringLiteral("Known mintage"), -1); + static const QString mintage(QStringLiteral("mintage")); + if(idx > -1 && optionalFields().contains(mintage)) { + entry_->setField(mintage, resultList_.at(idx).toString()); + } + + idx = m_colnectFields.value(QStringLiteral("Description"), -1); + static const QString desc(QStringLiteral("description")); + if(idx > -1 && optionalFields().contains(desc)) { + entry_->setField(desc, resultList_.at(idx).toString()); + } + + idx = m_colnectFields.value(QStringLiteral("FrontPicture"), -1); + if(idx > -1 && optionalFields().contains(QStringLiteral("obverse"))) { + entry_->setField(QStringLiteral("obverse"), + imageUrl(resultList_.at(0).toString(), + resultList_.at(idx).toString())); + } + + idx = m_colnectFields.value(QStringLiteral("BackPicture"), -1); + if(idx > -1 && optionalFields().contains(QStringLiteral("reverse"))) { + entry_->setField(QStringLiteral("reverse"), + imageUrl(resultList_.at(0).toString(), + resultList_.at(idx).toString())); + } +} + +Tellico::Fetch::ConfigWidget* ColnectFetcher::configWidget(QWidget* parent_) const { + return new ColnectFetcher::ConfigWidget(parent_, this); +} + +QString ColnectFetcher::defaultName() { + return QStringLiteral("Colnect"); // no translation +} + +QString ColnectFetcher::defaultIcon() { + return favIcon("https://colnect.com";); +} + +Tellico::StringHash ColnectFetcher::allOptionalFields() { + StringHash hash; + // treat images as optional since Colnect doesn't break out different images for each year + hash[QStringLiteral("obverse")] = i18n("Obverse"); + hash[QStringLiteral("reverse")] = i18n("Reverse"); + hash[QStringLiteral("series")] = i18n("Series"); + /* TRANSLATORS: Mintage refers to the number of coins minted */ + hash[QStringLiteral("mintage")] = i18n("Mintage"); + hash[QStringLiteral("description")] = i18n("Description"); + return hash; +} + +// Colnect specific method of turning name text into a slug +// $str = html_entity_decode($str, ENT_QUOTES, 'UTF-8'); +// $str = preg_replace('/&[^;]+;/', '_', $str); # change HTML elements to underscore +// $str = str_replace(array('.', '"', '>', '<', '\\', ':', '/', '?', '#', '[', ']', '@', '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '='), '', $str); +// $str = preg_replace('/[\s_]+/', '_', $str); # any space sequence becomes a single underscore +// $str = trim($str, '_'); # trim underscores +QString ColnectFetcher::URLize(const QString& name_) { + QString slug = name_; + static const QString underscore(QStringLiteral("_")); + static const QRegularExpression htmlElements(QStringLiteral("&[^;]+;")); + static const QRegularExpression toRemove(QStringLiteral("[.\"><\\:/?#\\[\\]@!$&'()*+,;=]")); + static const QRegularExpression spaces(QStringLiteral("\\s")); + slug.replace(htmlElements, underscore); + slug.remove(toRemove); + slug.replace(spaces, underscore); + while(slug.startsWith(underscore)) slug = slug.mid(1); + while(slug.endsWith(underscore)) slug.chop(1); + return slug; +} + +QString ColnectFetcher::imageUrl(const QString& name_, const QString& id_) { + const QString nameSlug = URLize(name_); + const int id = id_.toInt(); + QUrl u(QString::fromLatin1(COLNECT_IMAGE_URL)); + // uses 't' for thumbnail, use 'f' for full-size + u.setPath(QString::fromLatin1("/t/%1/%2/%3.jpg") + .arg(id / 1000) + .arg(id % 1000, 3, 10, QLatin1Char('0')) + .arg(nameSlug)); +// myDebug() << "Image url:" << u; + return u.toString(); +} + +void ColnectFetcher::readDataList() { +// myDebug() << "Reading Colnect fields"; + QUrl u(QString::fromLatin1(COLNECT_API_URL)); + // Colnect API calls are encoded as a path + QString query(QLatin1Char('/') + m_locale + QStringLiteral("/fields/cat/coins/")); + u.setPath(u.path() + query); + +// myDebug() << "Reading" << u; + const QByteArray data = FileHandler::readDataFile(u, true); + QJsonDocument doc = QJsonDocument::fromJson(data); + if(doc.isNull()) { + myDebug() << "null JSON document in colnect fields"; + return; + } + QVariantList resultList = doc.array().toVariantList(); + if(resultList.isEmpty()) { + myDebug() << "no colnect field results"; + return; + } + m_colnectFields.clear(); + for(int i = 0; i < resultList.size(); ++i) { + m_colnectFields.insert(resultList.at(i).toString(), i); +// if(i == 5) myDebug() << m_colnectFields; + } +// myDebug() << "Number of Colnect fields:" << m_colnectFields.count(); +} + +ColnectFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const ColnectFetcher* fetcher_) + : Fetch::ConfigWidget(parent_) { + QGridLayout* l = new QGridLayout(optionsWidget()); + l->setSpacing(4); + l->setColumnStretch(1, 10); + + int row = -1; + + QLabel* label = new QLabel(i18n("Language: "), optionsWidget()); + l->addWidget(label, ++row, 0); + m_langCombo = new GUI::ComboBox(optionsWidget()); + +#define LANG_ITEM(NAME, CY, ISO) \ + m_langCombo->addItem(QIcon(QStandardPaths::locate(QStandardPaths::GenericDataLocation, \ + QStringLiteral("kf5/locale/countries/" CY "/flag.png"))), \ + i18nc("Language", NAME), \ + QLatin1String(ISO)); + LANG_ITEM("English", "us", "en"); + LANG_ITEM("French", "fr", "fr"); + LANG_ITEM("German", "de", "de"); + LANG_ITEM("Spanish", "es", "es"); +#undef LANG_ITEM + + void (GUI::ComboBox::* activatedInt)(int) = &GUI::ComboBox::activated; + connect(m_langCombo, activatedInt, this, &ConfigWidget::slotSetModified); + connect(m_langCombo, activatedInt, this, &ConfigWidget::slotLangChanged); + l->addWidget(m_langCombo, row, 1); + label->setBuddy(m_langCombo); + + l->setRowStretch(++row, 10); + + // now add additional fields widget + addFieldsWidget(ColnectFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList()); + + if(fetcher_) { + m_langCombo->setCurrentData(fetcher_->m_locale); + } +} + +void ColnectFetcher::ConfigWidget::saveConfigHook(KConfigGroup& config_) { + const QString lang = m_langCombo->currentData().toString(); + config_.writeEntry("Locale", lang); +} + +QString ColnectFetcher::ConfigWidget::preferredName() const { + return QString::fromLatin1("Colnect (%1)").arg(m_langCombo->currentText()); +} + +void ColnectFetcher::ConfigWidget::slotLangChanged() { + emit signalName(preferredName()); +} diff --git a/src/fetch/colnectfetcher.h b/src/fetch/colnectfetcher.h new file mode 100644 index 00000000..19764f4b --- /dev/null +++ b/src/fetch/colnectfetcher.h @@ -0,0 +1,132 @@ +/*************************************************************************** + Copyright (C) 2019 Robby Stephenson <[email protected]> + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License or (at your option) version 3 or any later version * + * accepted by the membership of KDE e.V. (or its successor approved * + * by the membership of KDE e.V.), which shall act as a proxy * + * defined in Section 14 of version 3 of the license. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see <http://www.gnu.org/licenses/>. * + * * + ***************************************************************************/ + +#ifndef TELLICO_COLNECTFETCHER_H +#define TELLICO_COLNECTFETCHER_H + +#include "fetcher.h" +#include "configwidget.h" +#include "../datavectors.h" + +#include <QPointer> +#include <QVariantMap> + +class KJob; +namespace KIO { + class StoredTransferJob; +} + +class ColnectFetcherTest; +namespace Tellico { + + namespace GUI { + class ComboBox; + } + + namespace Fetch { + +/** + * A fetcher for colnect.org + * + * @author Robby Stephenson + */ +class ColnectFetcher : public Fetcher { +Q_OBJECT + +friend class ::ColnectFetcherTest; + +public: + /** + */ + ColnectFetcher(QObject* parent); + /** + */ + virtual ~ColnectFetcher(); + + /** + */ + virtual QString source() const Q_DECL_OVERRIDE; + virtual QString attribution() const Q_DECL_OVERRIDE; + virtual bool isSearching() const Q_DECL_OVERRIDE { return m_started; } + virtual bool canSearch(FetchKey k) const Q_DECL_OVERRIDE; + virtual void stop() Q_DECL_OVERRIDE; + virtual Data::EntryPtr fetchEntryHook(uint uid) Q_DECL_OVERRIDE; + virtual Type type() const Q_DECL_OVERRIDE { return Colnect; } + virtual bool canFetch(int type) const Q_DECL_OVERRIDE; + virtual void readConfigHook(const KConfigGroup& config) Q_DECL_OVERRIDE; + + /** + * Returns a widget for modifying the fetcher's config. + */ + virtual Fetch::ConfigWidget* configWidget(QWidget* parent) const Q_DECL_OVERRIDE; + + class ConfigWidget; + friend class ConfigWidget; + + static QString defaultName(); + static QString defaultIcon(); + static StringHash allOptionalFields(); + +private Q_SLOTS: + void slotComplete(KJob* job); + +private: + static QString URLize(const QString& name); + static QString imageUrl(const QString& name, const QString& id); + + virtual void search() Q_DECL_OVERRIDE; + virtual FetchRequest updateRequest(Data::EntryPtr entry) Q_DECL_OVERRIDE; + void populateEntry(Data::EntryPtr entry, const QVariantList& resultList); + + void readDataList(); + + QHash<uint, Data::EntryPtr> m_entries; + + bool m_started; + QString m_locale; + QPointer<KIO::StoredTransferJob> m_job; + QString m_year; + + // map from field name to position in result list + QHash<QString, int> m_colnectFields; +}; + +class ColnectFetcher::ConfigWidget : public Fetch::ConfigWidget { +Q_OBJECT + +public: + explicit ConfigWidget(QWidget* parent_, const ColnectFetcher* fetcher = nullptr); + virtual void saveConfigHook(KConfigGroup&) Q_DECL_OVERRIDE; + virtual QString preferredName() const Q_DECL_OVERRIDE; + +private Q_SLOTS: + void slotLangChanged(); + +private: + GUI::ComboBox* m_langCombo; +}; + + } // end namespace +} // end namespace +#endif diff --git a/src/fetch/fetch.h b/src/fetch/fetch.h index de679d59..22d0878b 100644 --- a/src/fetch/fetch.h +++ b/src/fetch/fetch.h @@ -103,7 +103,8 @@ enum Type { Kino, MobyGames, ComicVine, - KinoTeatr + KinoTeatr, + Colnect }; } diff --git a/src/fetch/fetcherinitializer.cpp b/src/fetch/fetcherinitializer.cpp index 21edf629..d46ca09a 100644 --- a/src/fetch/fetcherinitializer.cpp +++ b/src/fetch/fetcherinitializer.cpp @@ -71,6 +71,7 @@ #include "mobygamesfetcher.h" #include "comicvinefetcher.h" #include "kinoteatrfetcher.h" +#include "colnectfetcher.h" /** * Ideally, I'd like these initializations to be in each cpp file for each collection type @@ -127,6 +128,7 @@ Tellico::Fetch::FetcherInitializer::FetcherInitializer() { RegisterFetcher<Fetch::MobyGamesFetcher> registerMobyGames(MobyGames); RegisterFetcher<Fetch::ComicVineFetcher> registerComicVine(ComicVine); RegisterFetcher<Fetch::KinoTeatrFetcher> registerTeatr(KinoTeatr); + RegisterFetcher<Fetch::ColnectFetcher> registerColnect(Colnect); Fetch::Manager::self()->loadFetchers(); } diff --git a/src/fetch/fetchresult.cpp b/src/fetch/fetchresult.cpp index 1f720a8e..df76ff47 100644 --- a/src/fetch/fetchresult.cpp +++ b/src/fetch/fetchresult.cpp @@ -115,6 +115,11 @@ QString FetchResult::makeDescription(Data::EntryPtr entry) { append(desc, entry, "appellation"); break; + case Data::Collection::Coin: + append(desc, entry, "country"); + append(desc, entry, "description"); + break; + default: myDebug() << "no description for collection type =" << entry->collection()->type(); break; diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 394a2b68..12c1fbe2 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -518,6 +518,14 @@ add_test(comicvinefetchertest comicvinefetchertest) ecm_mark_as_test(comicvinefetchertest) TARGET_LINK_LIBRARIES(comicvinefetchertest fetcherstest ${TELLICO_TEST_LIBS}) +add_executable(colnectfetchertest colnectfetchertest.cpp abstractfetchertest.cpp + ../fetch/colnectfetcher.cpp +) +ecm_mark_nongui_executable(colnectfetchertest) +add_test(colnectfetchertest colnectfetchertest) +ecm_mark_as_test(colnectfetchertest) +TARGET_LINK_LIBRARIES(colnectfetchertest fetcherstest ${TELLICO_TEST_LIBS}) + add_executable(crossreffetchertest crossreffetchertest.cpp abstractfetchertest.cpp ../fetch/crossreffetcher.cpp ) diff --git a/src/tests/colnectfetchertest.cpp b/src/tests/colnectfetchertest.cpp new file mode 100644 index 00000000..bdf7db88 --- /dev/null +++ b/src/tests/colnectfetchertest.cpp @@ -0,0 +1,130 @@ +/*************************************************************************** + Copyright (C) 2019 Robby Stephenson <[email protected]> + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License or (at your option) version 3 or any later version * + * accepted by the membership of KDE e.V. (or its successor approved * + * by the membership of KDE e.V.), which shall act as a proxy * + * defined in Section 14 of version 3 of the license. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see <http://www.gnu.org/licenses/>. * + * * + ***************************************************************************/ + +#undef QT_NO_CAST_FROM_ASCII + +#include "colnectfetchertest.h" + +#include "../fetch/colnectfetcher.h" +#include "../entry.h" +#include "../collections/coincollection.h" +#include "../collectionfactory.h" +#include "../images/imagefactory.h" +#include "../fieldformat.h" +#include "../fetch/fetcherjob.h" + +#include <KConfig> +#include <KConfigGroup> + +#include <QTest> + +QTEST_GUILESS_MAIN( ColnectFetcherTest ) + +ColnectFetcherTest::ColnectFetcherTest() : AbstractFetcherTest() { +} + +void ColnectFetcherTest::initTestCase() { + Tellico::ImageFactory::init(); + Tellico::RegisterCollection<Tellico::Data::CoinCollection> registerMe(Tellico::Data::Collection::Coin, "coin"); +} + +void ColnectFetcherTest::testSlug() { + // test the implementation of the Colnect slug derivation + QFETCH(QString, input); + QFETCH(QString, slug); + + QCOMPARE(Tellico::Fetch::ColnectFetcher::URLize(input), slug); +} + +void ColnectFetcherTest::testSlug_data() { + QTest::addColumn<QString>("input"); + QTest::addColumn<QString>("slug"); + + QTest::newRow("basic") << QStringLiteral("input") << QStringLiteral("input"); + QTest::newRow("Aus1$") << QStringLiteral("1 Dollar (50 Years Moonlanding)") << QStringLiteral("1_Dollar_50_Years_Moonlanding"); +} + +void ColnectFetcherTest::testRaw() { + KConfig config(QFINDTESTDATA("tellicotest.config"), KConfig::SimpleConfig); + QString groupName = QStringLiteral("colnect"); + if(!config.hasGroup(groupName)) { + QSKIP("This test requires a config file.", SkipAll); + } + KConfigGroup cg(&config, groupName); + + Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Coin, + Tellico::Fetch::Raw, + QStringLiteral("147558")); + Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::ColnectFetcher(this)); + fetcher->readConfig(cg, cg.name()); + + Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1); + + QCOMPARE(results.size(), 1); + Tellico::Data::EntryPtr entry = results.at(0); + + QCOMPARE(entry->field(QStringLiteral("year")), QStringLiteral("2019")); + QCOMPARE(entry->field(QStringLiteral("country")), QStringLiteral("Australia")); + QCOMPARE(entry->field(QStringLiteral("denomination")), QStringLiteral("$1.00")); + QCOMPARE(entry->field(QStringLiteral("currency")), QStringLiteral("$ - Australian dollar")); + QCOMPARE(entry->field(QStringLiteral("series")), QStringLiteral("1952~Today - Elizabeth II")); + QCOMPARE(entry->field(QStringLiteral("mintage")), QStringLiteral("25000")); + QVERIFY(!entry->field(QStringLiteral("description")).isEmpty()); + QVERIFY(!entry->field(QStringLiteral("obverse")).isEmpty()); + QVERIFY(!entry->field(QStringLiteral("obverse")).contains(QLatin1Char('/'))); + QVERIFY(!entry->field(QStringLiteral("reverse")).isEmpty()); + QVERIFY(!entry->field(QStringLiteral("reverse")).contains(QLatin1Char('/'))); +} + +void ColnectFetcherTest::testSacagawea() { + KConfig config(QFINDTESTDATA("tellicotest.config"), KConfig::SimpleConfig); + QString groupName = QStringLiteral("colnect"); + if(!config.hasGroup(groupName)) { + QSKIP("This test requires a config file.", SkipAll); + } + KConfigGroup cg(&config, groupName); + + Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Coin, + Tellico::Fetch::Keyword, + QStringLiteral("2007 Sacagawea")); + Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::ColnectFetcher(this)); + fetcher->readConfig(cg, cg.name()); + + Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1); + + QCOMPARE(results.size(), 1); + Tellico::Data::EntryPtr entry = results.at(0); + + QCOMPARE(entry->field(QStringLiteral("year")), QStringLiteral("2007")); + QCOMPARE(entry->field(QStringLiteral("country")), QStringLiteral("United States of America")); + QCOMPARE(entry->field(QStringLiteral("denomination")), QStringLiteral("$1.00")); + QCOMPARE(entry->field(QStringLiteral("currency")), QStringLiteral("$ - United States dollar")); + QCOMPARE(entry->field(QStringLiteral("series")), QStringLiteral("B06a - Eisenhower, Anthony & Sacagawea Dollar")); + QCOMPARE(entry->field(QStringLiteral("mintage")), QStringLiteral("1497251077")); + QVERIFY(!entry->field(QStringLiteral("description")).isEmpty()); + QVERIFY(!entry->field(QStringLiteral("obverse")).isEmpty()); + QVERIFY(!entry->field(QStringLiteral("obverse")).contains(QLatin1Char('/'))); + QVERIFY(!entry->field(QStringLiteral("reverse")).isEmpty()); + QVERIFY(!entry->field(QStringLiteral("reverse")).contains(QLatin1Char('/'))); +} diff --git a/src/fetch/fetch.h b/src/tests/colnectfetchertest.h similarity index 57% copy from src/fetch/fetch.h copy to src/tests/colnectfetchertest.h index de679d59..40a27436 100644 --- a/src/fetch/fetch.h +++ b/src/tests/colnectfetchertest.h @@ -1,5 +1,5 @@ /*************************************************************************** - Copyright (C) 2003-2009 Robby Stephenson <[email protected]> + Copyright (C) 2019 Robby Stephenson <[email protected]> ***************************************************************************/ /*************************************************************************** @@ -22,91 +22,22 @@ * * ***************************************************************************/ -#ifndef TELLICO_FETCH_H -#define TELLICO_FETCH_H +#ifndef COLNECTFETCHERTEST_H +#define COLNECTFETCHERTEST_H -namespace Tellico { - namespace Fetch { +#include "abstractfetchertest.h" -/** - * FetchFirst must be first, and the rest must follow consecutively in value. - * FetchLast must be last! - */ -enum FetchKey { - FetchFirst = 0, - Title, - Person, - ISBN, - UPC, - Keyword, - DOI, - ArxivID, - PubmedID, - LCCN, - Raw, - ExecUpdate, - FetchLast -}; +class ColnectFetcherTest : public AbstractFetcherTest { +Q_OBJECT +public: + ColnectFetcherTest(); -// real ones must start at 0! -enum Type { - Unknown = -1, - Amazon = 0, - IMDB, - Z3950, - SRU, - Entrez, - ExecExternal, - Yahoo, // Removed - AnimeNfo, - IBS, - ISBNdb, - GCstarPlugin, - CrossRef, - Citebase, // Removed - Arxiv, - Bibsonomy, - GoogleScholar, - Discogs, - WineCom, - TheMovieDB, - MusicBrainz, - GiantBomb, - OpenLibrary, - Multiple, - Freebase, // Removed - DVDFr, - Filmaster, - Douban, - BiblioShare, - MovieMeter, - GoogleBook, - MAS, // Removed - Springer, - Allocine, - ScreenRush, // Removed - FilmStarts, // Removed - SensaCine, // Removed - Beyazperde, // Removed - HathiTrust, - TheGamesDB, - DBLP, - VNDB, - MRLookup, - BoardGameGeek, - Bedetheque, - OMDB, - KinoPoisk, - VideoGameGeek, - DBC, - IGDB, - Kino, - MobyGames, - ComicVine, - KinoTeatr +private Q_SLOTS: + void initTestCase(); + void testSlug(); + void testSlug_data(); + void testRaw(); + void testSacagawea(); }; - } -} - #endif diff --git a/src/tests/tellicotest.config b/src/tests/tellicotest.config index 61f85822..c742360d 100644 --- a/src/tests/tellicotest.config +++ b/src/tests/tellicotest.config @@ -75,3 +75,6 @@ Custom Fields=comicvine,colorist [kinoteatr] Custom Fields=origtitle,kinoteatr + +[colnect] +Custom Fields=obverse,reverse,series,mintage,description
