Git commit 6ac873862e9e44f6f6ade133d46917b3c86b4a26 by Robby Stephenson. Committed on 16/01/2023 at 21:35. Pushed by rstephenson into branch 'master'.
Add data source for FilmAffinity, with Spanish and English M +4 -0 ChangeLog M +8 -0 doc/configuration.docbook M +1 -0 src/fetch/CMakeLists.txt M +2 -1 src/fetch/fetch.h M +2 -0 src/fetch/fetcherinitializer.cpp A +518 -0 src/fetch/filmaffinityfetcher.cpp [License: GPL (v2/3)] A +131 -0 src/fetch/filmaffinityfetcher.h [License: GPL (v2/3)] M +6 -0 src/tests/CMakeLists.txt A +173 -0 src/tests/filmaffinityfetchertest.cpp [License: GPL (v2/3)] C +19 -90 src/tests/filmaffinityfetchertest.h [from: src/fetch/fetch.h - 055% similarity] https://invent.kde.org/office/tellico/commit/6ac873862e9e44f6f6ade133d46917b3c86b4a26 diff --git a/ChangeLog b/ChangeLog index c4bb1d9d..a3be2db0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2023-01-16 Robby Stephenson <ro...@periapsis.org> + + * Added data source for FilmAffinity. + 2023-01-12 Robby Stephenson <ro...@periapsis.org> * Fixed bug with timing of multisource config read (Bug 461861). diff --git a/doc/configuration.docbook b/doc/configuration.docbook index 450ecda4..47dc003f 100644 --- a/doc/configuration.docbook +++ b/doc/configuration.docbook @@ -168,6 +168,7 @@ while the full list is <ulink url="https://tellico-project.org/data-sources";>ava <listitem><simpara><link linkend="allocine">AlloCiné</link>,</simpara></listitem> <listitem><simpara><link linkend="tmdb">TheMovieDB.org</link>,</simpara></listitem> <listitem><simpara>the <link linkend="omdb">Open Movie Database</link>,</simpara></listitem> +<listitem><simpara><link linkend="filmaffinity">FilmAffinity</link>,</simpara></listitem> <!-- comics --> <listitem><simpara><link linkend="bedetheque">BDGest</link>,</simpara></listitem> <listitem><simpara><link linkend="comicvine">Comic Vine</link>,</simpara></listitem> @@ -366,6 +367,13 @@ The <ulink url="http://www.imdb.com";>Internet Movie Database</ulink> provides in </para> </sect3> +<sect3 id="filmaffinity"> +<title>FilmAffinity</title> +<para> +<ulink url="https://filmaffinity.com";>FilmAffinity</ulink> is an independent film site. +</para> +</sect3> + </sect2> <!-- start of music sources --> diff --git a/src/fetch/CMakeLists.txt b/src/fetch/CMakeLists.txt index 9c6a4d83..359c1236 100644 --- a/src/fetch/CMakeLists.txt +++ b/src/fetch/CMakeLists.txt @@ -30,6 +30,7 @@ SET(fetch_STAT_SRCS fetchmanager.cpp fetchrequest.cpp fetchresult.cpp + filmaffinityfetcher.cpp filmasterfetcher.cpp gaminghistoryfetcher.cpp gcstarpluginfetcher.cpp diff --git a/src/fetch/fetch.h b/src/fetch/fetch.h index 4cbbd4bd..0905962b 100644 --- a/src/fetch/fetch.h +++ b/src/fetch/fetch.h @@ -110,7 +110,8 @@ enum Type { UPCItemDb, TheTVDB, RPGGeek, - GamingHistory + GamingHistory, + FilmAffinity }; } diff --git a/src/fetch/fetcherinitializer.cpp b/src/fetch/fetcherinitializer.cpp index 454ba475..e042a167 100644 --- a/src/fetch/fetcherinitializer.cpp +++ b/src/fetch/fetcherinitializer.cpp @@ -77,6 +77,7 @@ #include "thetvdbfetcher.h" #include "rpggeekfetcher.h" #include "gaminghistoryfetcher.h" +#include "filmaffinityfetcher.h" /** * Ideally, I'd like these initializations to be in each cpp file for each collection type @@ -132,6 +133,7 @@ Tellico::Fetch::FetcherInitializer::FetcherInitializer() { RegisterFetcher<Fetch::TheTVDBFetcher> registerTheTVDB(TheTVDB); RegisterFetcher<Fetch::RPGGeekFetcher> registerRPGGeek(RPGGeek); RegisterFetcher<Fetch::GamingHistoryFetcher> registerGamingHistory(GamingHistory); + RegisterFetcher<Fetch::FilmAffinityFetcher> registerFilmAffinity(FilmAffinity); // these data sources depend on being able to import bibtex #ifdef ENABLE_BTPARSE diff --git a/src/fetch/filmaffinityfetcher.cpp b/src/fetch/filmaffinityfetcher.cpp new file mode 100644 index 00000000..3f9bd39f --- /dev/null +++ b/src/fetch/filmaffinityfetcher.cpp @@ -0,0 +1,518 @@ +/*************************************************************************** + Copyright (C) 2023 Robby Stephenson <ro...@periapsis.org> + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License or (at your option) version 3 or any later version * + * accepted by the membership of KDE e.V. (or its successor approved * + * by the membership of KDE e.V.), which shall act as a proxy * + * defined in Section 14 of version 3 of the license. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see <http://www.gnu.org/licenses/>. * + * * + ***************************************************************************/ + +#include "filmaffinityfetcher.h" +#include "../utils/guiproxy.h" +#include "../utils/string_utils.h" +#include "../collections/videocollection.h" +#include "../entry.h" +#include "../fieldformat.h" +#include "../core/filehandler.h" +#include "../images/imagefactory.h" +#include "../gui/combobox.h" +#include "../tellico_debug.h" + +#include <KLocalizedString> +#include <KIO/Job> +#include <KJobUiDelegate> +#include <KJobWidgets/KJobWidgets> + +#include <QRegularExpression> +#include <QLabel> +#include <QFile> +#include <QTextStream> +#include <QGridLayout> +#include <QSpinBox> +#include <QUrlQuery> +#include <QStandardPaths> + +namespace { + static const char* FILMAFFINITY_SEARCH_URL = "https://www.filmaffinity.com";; + static const uint FILMAFFINITY_DEFAULT_CAST_SIZE = 10; +} + +using namespace Tellico; +using Tellico::Fetch::FilmAffinityFetcher; + +FilmAffinityFetcher::FilmAffinityFetcher(QObject* parent_) + : Fetcher(parent_), m_started(false), m_locale(ES), m_numCast(FILMAFFINITY_DEFAULT_CAST_SIZE) { +} + +FilmAffinityFetcher::~FilmAffinityFetcher() { +} + +// static +const FilmAffinityFetcher::LocaleData& FilmAffinityFetcher::localeData(int locale_) { + Q_ASSERT(locale_ >= 0); + Q_ASSERT(locale_ < 2); + static LocaleData dataVector[6] = { + { + QStringLiteral("es"), + QStringLiteral("(Serie de TV)"), + QString::fromUtf8("Año"), + QStringLiteral("Título original"), + QStringLiteral("País"), + QString::fromUtf8("Duración"), + QString::fromUtf8("Dirección"), + QStringLiteral("Reparto"), + QString::fromUtf8("Género"), + QStringLiteral("Guion"), + QStringLiteral("Historia:"), + QString::fromUtf8("Compañías"), + QStringLiteral("Distribuidora"), + QStringLiteral("Emitida por:"), + QString::fromUtf8("Música"), + QStringLiteral("Sinopsis") + }, + { + QStringLiteral("us"), + QStringLiteral("(TV Series)"), + QStringLiteral("Year"), + QStringLiteral("Original title"), + QStringLiteral("Country"), + QStringLiteral("Running time"), + QStringLiteral("Director"), + QStringLiteral("Cast"), + QStringLiteral("Genre"), + QStringLiteral("Screenwriter"), + QStringLiteral("Story:"), + QStringLiteral("Producer"), + QStringLiteral("Distributor:"), + QStringLiteral("Broadcast by:"), + QStringLiteral("Music"), + QStringLiteral("Synopsis") + } + }; + + return dataVector[qBound(0, locale_, static_cast<int>(sizeof(dataVector)/sizeof(LocaleData)))]; +} + +QString FilmAffinityFetcher::source() const { + return m_name.isEmpty() ? defaultName() : m_name; +} + +bool FilmAffinityFetcher::canFetch(int type) const { + return type == Data::Collection::Video; +} + +bool FilmAffinityFetcher::canSearch(Fetch::FetchKey k) const { + return k == Title; +} + +void FilmAffinityFetcher::readConfigHook(const KConfigGroup& config_) { + const int locale = config_.readEntry("Locale", int(ES)); + m_locale = static_cast<Locale>(locale); + m_numCast = config_.readEntry("Max Cast", FILMAFFINITY_DEFAULT_CAST_SIZE); +} + +void FilmAffinityFetcher::search() { + m_started = true; + m_matches.clear(); + + QUrl u(QString::fromLatin1(FILMAFFINITY_SEARCH_URL)); + u.setPath(QLatin1String("/") + localeData(m_locale).siteSlug + QLatin1String("/advsearch.php")); + QString searchValue = request().value(); + QUrlQuery q; + // extract the year from the end of the search string, accept the posible corner case of a movie + // having some other year in the title? + QRegularExpression yearRx(QStringLiteral("\\s(19|20)\\d\\d$")); + auto match = yearRx.match(searchValue); + if(match.hasMatch()) { + searchValue.remove(match.captured()); + const auto& year = match.captured().simplified(); + q.addQueryItem(QStringLiteral("fromyear"), year); + q.addQueryItem(QStringLiteral("toyear"), year); + } + q.addQueryItem(QStringLiteral("stext"), searchValue); + + switch(request().key()) { + case Title: + //q.addQueryItem(QStringLiteral("year"), QStringLiteral("yes")); + q.addQueryItem(QStringLiteral("stype[]"), QLatin1String("title")); + break; + + default: + myWarning() << "key not recognized: " << request().key(); + stop(); + return; + } + u.setQuery(q); +// myDebug() << "url: " << u.url(); + + m_job = KIO::storedGet(u, KIO::NoReload, KIO::HideProgressInfo); + KJobWidgets::setWindow(m_job, GUI::Proxy::widget()); + connect(m_job.data(), &KJob::result, this, &FilmAffinityFetcher::slotComplete); +} + +void FilmAffinityFetcher::stop() { + if(!m_started) { + return; + } + + if(m_job) { + m_job->kill(); + m_job = nullptr; + } + m_started = false; + emit signalDone(this); +} + +void FilmAffinityFetcher::slotComplete(KJob*) { + if(m_job->error()) { + m_job->uiDelegate()->showErrorMessage(); + stop(); + return; + } + + QByteArray data = m_job->data(); + if(data.isEmpty()) { + myDebug() << "no data"; + stop(); + return; + } + + const QString output = Tellico::decodeHTML(data); +#if 0 + myWarning() << "Remove debug from filmaffinityfetcher.cpp"; + QFile f(QStringLiteral("/tmp/test1.html")); + if(f.open(QIODevice::WriteOnly)) { + QTextStream t(&f); + t.setCodec("UTF-8"); + t << output; + } + f.close(); +#endif + + // look for a specific div, with an href and title, sometime uses single-quote, sometimes double-quotes + QRegularExpression resultRx(QStringLiteral("<div class=\"fa-shadow adv-search-item\">(.+?)<div class=\"mc-actions\">"), + QRegularExpression::DotMatchesEverythingOption); + QRegularExpression titleRx(QStringLiteral("<a\\s+href=\"(.+?)\"\\s+title=\"(.+?)\">(.+?)<img")); + // the year is within the title text as a 4-digit number, starting with 1 or 2 + QRegularExpression yearRx(QStringLiteral("\\(([12]\\d\\d\\d)\\)")); + + QString href, title, year; + QRegularExpressionMatchIterator i = resultRx.globalMatch(output); + while(i.hasNext() && m_started) { + auto topMatch = i.next(); + auto anchorMatch = titleRx.match(topMatch.captured(1)); + if(anchorMatch.hasMatch()) { + href = anchorMatch.captured(1); + title = anchorMatch.captured(2).trimmed(); + auto yearMatch = yearRx.match(anchorMatch.captured(3)); + if(yearMatch.hasMatch()) { + year = yearMatch.captured(1); + } + } + if(!href.isEmpty()) { + QUrl url(QString::fromLatin1(FILMAFFINITY_SEARCH_URL)); + url = url.resolved(QUrl(href)); +// myDebug() << url << title << year; + FetchResult* r = new FetchResult(this, title, year); + m_matches.insert(r->uid, url); + emit signalResultFound(r); + } + } + + // since the fetch is done, don't worry about holding the job pointer + m_job = nullptr; + stop(); +} + +Tellico::Data::EntryPtr FilmAffinityFetcher::fetchEntryHook(uint uid_) { + // if we already grabbed this one, then just pull it out of the dict + Data::EntryPtr entry = m_entries[uid_]; + if(entry) { + return entry; + } + + QUrl url = m_matches[uid_]; + if(url.isEmpty()) { + myWarning() << "no url in map"; + return Data::EntryPtr(); + } + + const QString results = Tellico::decodeHTML(FileHandler::readDataFile(url, true)); + if(results.isEmpty()) { + myDebug() << "no text results"; + return Data::EntryPtr(); + } + +#if 0 + myDebug() << url.url(); + myWarning() << "Remove debug2 from filmaffinityfetcher.cpp"; + QFile f(QStringLiteral("/tmp/test-filmaffinity.html")); + if(f.open(QIODevice::WriteOnly)) { + QTextStream t(&f); + t.setCodec("UTF-8"); + t << results; + } + f.close(); +#endif + + entry = parseEntry(results); + if(!entry) { + myDebug() << "error in processing entry"; + return Data::EntryPtr(); + } + + const QString fa = QStringLiteral("filmaffinity"); + if(optionalFields().contains(fa)) { + Data::FieldPtr field(new Data::Field(fa, i18n("FilmAffinity Link"), Data::Field::URL)); + field->setCategory(i18n("General")); + entry->collection()->addField(field); + entry->setField(fa, url.url()); + } + + m_entries.insert(uid_, entry); // keep for later + return entry; +} + +Tellico::Data::EntryPtr FilmAffinityFetcher::parseEntry(const QString& str_) { + Data::CollPtr coll(new Data::VideoCollection(true)); + Data::EntryPtr entry(new Data::Entry(coll)); + coll->addEntries(entry); + + const LocaleData& data = localeData(m_locale); + + QRegularExpression titleRx(QStringLiteral("<span itemprop=\"name\">(.+?)</span")); + QRegularExpressionMatch match = titleRx.match(str_); + if(match.hasMatch()) { + // remove anything in parentheses + QString title = match.captured(1).simplified(); + title.remove(data.tvSeries); + title = title.trimmed(); + entry->setField(QStringLiteral("title"), title); + } + + const QString origtitle = QStringLiteral("origtitle"); + QRegularExpression tagRx(QStringLiteral("<.+?>")); + QRegularExpression spanRx(QStringLiteral("<span.*?>(.+?)</span")); + QRegularExpression defRx(QStringLiteral("<dt>(.+?)</dt>\\s*?<dd.*?>(.+?)</dd>"), + QRegularExpression::DotMatchesEverythingOption); + QRegularExpressionMatchIterator i = defRx.globalMatch(str_); + while(i.hasNext()) { + auto match = i.next(); + const auto& term = match.captured(1); + if(term == data.year) { + entry->setField(QStringLiteral("year"), match.captured(2).trimmed()); + } else if(term == data.origTitle && + optionalFields().contains(origtitle)) { + Data::FieldPtr f(new Data::Field(origtitle, i18n("Original Title"))); + f->setFormatType(FieldFormat::FormatTitle); + coll->addField(f); + // might have an aka in a span + QString oTitle = match.captured(2); + const int start = oTitle.indexOf(QLatin1String("<span")); + if(start > -1) oTitle = oTitle.left(start); + entry->setField(origtitle, oTitle.remove(tagRx).simplified()); + } else if(term == data.runningTime) { + QRegularExpression timeRx(QStringLiteral("\\d+")); + auto timeMatch = timeRx.match(match.captured(2)); + if(timeMatch.hasMatch()) { + entry->setField(QStringLiteral("running-time"), timeMatch.captured()); + } + } else if(term == data.country) { + QRegularExpression countryRx(QStringLiteral("alt=\"(.+?)\"")); + auto countryMatch = countryRx.match(match.captured(2)); + if(countryMatch.hasMatch()) { + entry->setField(QStringLiteral("nationality"), countryMatch.captured(1)); + } + } else if(term == data.director) { + QStringList directors; + auto iSpan = spanRx.globalMatch(match.captured(2)); + while(iSpan.hasNext()) { + auto spanMatch = iSpan.next(); + directors += spanMatch.captured(1).remove(tagRx).simplified(); + } + if(!directors.isEmpty()) { + entry->setField(QStringLiteral("director"), directors.join(FieldFormat::delimiterString())); + } + } else if(term == data.cast) { + QStringList cast; + const auto& captured = match.captured(2); + // only read up to thie hidden credits + const auto end = captured.indexOf(QLatin1String("hidden-credit")); + auto iSpan = spanRx.globalMatch(captured.left(end)); + while(iSpan.hasNext() && cast.size() < m_numCast) { + auto spanMatch = iSpan.next(); + cast += spanMatch.captured(1).remove(tagRx).simplified(); + } + if(!cast.isEmpty()) { + entry->setField(QStringLiteral("cast"), cast.join(FieldFormat::rowDelimiterString())); + } + } else if(term == data.genre) { + QStringList genres; + auto iSpan = spanRx.globalMatch(match.captured(2)); + while(iSpan.hasNext()) { + auto spanMatch = iSpan.next(); + genres += spanMatch.captured(1).remove(tagRx).simplified(); + } + if(!genres.isEmpty()) { + entry->setField(QStringLiteral("genre"), genres.join(FieldFormat::delimiterString())); + } + } else if(term == data.writer) { + QStringList writers; + const auto& captured = match.captured(2); + // skip ahead to "Story" + const auto start = captured.indexOf(data.story); + auto iSpan = spanRx.globalMatch(captured.mid(qMax(0,start))); + while(iSpan.hasNext()) { + auto spanMatch = iSpan.next(); + writers += spanMatch.captured(1).remove(tagRx).simplified(); + } + if(!writers.isEmpty()) { + entry->setField(QStringLiteral("writer"), writers.join(FieldFormat::delimiterString())); + } + } else if(term == data.producer) { + // producer seems to be all the studio, use distributor as the main + QStringList studios; + const auto& captured = match.captured(2); + // skip ahead to "Story" + const auto start1 = captured.indexOf(data.distributor); + const auto start2 = captured.indexOf(data.broadcast); + auto iSpan = spanRx.globalMatch(captured.mid(qMax(0,qMax(start1,start2)))); + while(iSpan.hasNext()) { + auto spanMatch = iSpan.next(); + studios += spanMatch.captured(1).remove(tagRx).simplified(); + } + if(!studios.isEmpty()) { + entry->setField(QStringLiteral("studio"), studios.join(FieldFormat::delimiterString())); + } + } else if(term == data.music) { + entry->setField(QStringLiteral("composer"), match.captured(2).remove(tagRx).trimmed()); + } else if(term == data.plot) { + entry->setField(QStringLiteral("plot"), match.captured(2).trimmed()); + } + } + + QString cover; + QRegularExpression coverRx(QStringLiteral("<img\\s.*?itemprop=\"image\".+?src=\"(.+?)\".*?>")); + match = coverRx.match(str_); + if(match.hasMatch()) { + cover = match.captured(1); + } else { + coverRx.setPattern(QStringLiteral("<meta property=\"og:image\" content=\"(.+?)\"")); + match = coverRx.match(str_); + if(match.hasMatch()) { + cover = match.captured(1); + } + } + if(!cover.isEmpty()) { +// myDebug() << "cover:" << cover; + const QString id = ImageFactory::addImage(QUrl::fromUserInput(cover), true /* quiet */); + if(id.isEmpty()) { + message(i18n("The cover image could not be loaded."), MessageHandler::Warning); + } + // empty image ID is ok + entry->setField(QStringLiteral("cover"), id); + } + + return entry; +} + +Tellico::Fetch::FetchRequest FilmAffinityFetcher::updateRequest(Data::EntryPtr entry_) { + QString t = entry_->field(QStringLiteral("title")); + if(!t.isEmpty()) { + return FetchRequest(Fetch::Title, t); + } + return FetchRequest(); +} + +Tellico::Fetch::ConfigWidget* FilmAffinityFetcher::configWidget(QWidget* parent_) const { + return new FilmAffinityFetcher::ConfigWidget(parent_); +} + +QString FilmAffinityFetcher::defaultName() { + return QStringLiteral("FilmAffinity"); +} + +QString FilmAffinityFetcher::defaultIcon() { + return favIcon("https://www.filmaffinity.com";); +} + +Tellico::StringHash FilmAffinityFetcher::allOptionalFields() { + StringHash hash; + hash[QStringLiteral("origtitle")] = i18n("Original Title"); + hash[QStringLiteral("filmaffinity")] = i18n("FilmAffinity Link"); + return hash; +} + +FilmAffinityFetcher::ConfigWidget::ConfigWidget(QWidget* parent_, const FilmAffinityFetcher* fetcher_) + : Fetch::ConfigWidget(parent_) { + QGridLayout* l = new QGridLayout(optionsWidget()); + l->setSpacing(4); + l->setColumnStretch(1, 10); + + int row = -1; + + QLabel* label = new QLabel(i18n("&Maximum cast: "), optionsWidget()); + l->addWidget(label, ++row, 0); + m_numCast = new QSpinBox(optionsWidget()); + m_numCast->setMaximum(99); + m_numCast->setMinimum(0); + m_numCast->setValue(FILMAFFINITY_DEFAULT_CAST_SIZE); +#if (QT_VERSION < QT_VERSION_CHECK(5, 14, 0)) + void (QSpinBox::* textChanged)(const QString&) = &QSpinBox::valueChanged; +#else + void (QSpinBox::* textChanged)(const QString&) = &QSpinBox::textChanged; +#endif + connect(m_numCast, textChanged, this, &ConfigWidget::slotSetModified); + l->addWidget(m_numCast, row, 1); + QString w = i18n("The list of cast members may include many people. Set the maximum number returned from the search."); + label->setWhatsThis(w); + m_numCast->setWhatsThis(w); + label->setBuddy(m_numCast); + + label = new QLabel(i18n("Language: "), optionsWidget()); + l->addWidget(label, ++row, 0); + m_localeCombo = new GUI::ComboBox(optionsWidget()); + QIcon iconES(QStandardPaths::locate(QStandardPaths::GenericDataLocation, + QStringLiteral("kf5/locale/countries/es/flag.png"))); + m_localeCombo->addItem(iconES, i18nc("Country", "Spain"), int(FilmAffinityFetcher::ES)); + QIcon iconUS(QStandardPaths::locate(QStandardPaths::GenericDataLocation, + QStringLiteral("kf5/locale/countries/us/flag.png"))); + m_localeCombo->addItem(iconUS, i18nc("Country", "USA"), int(FilmAffinityFetcher::US)); + void (GUI::ComboBox::* activatedInt)(int) = &GUI::ComboBox::activated; + connect(m_localeCombo, activatedInt, this, &ConfigWidget::slotSetModified); + l->addWidget(m_localeCombo, row, 1); + label->setBuddy(m_localeCombo); + + l->setRowStretch(++row, 10); + + addFieldsWidget(FilmAffinityFetcher::allOptionalFields(), fetcher_ ? fetcher_->optionalFields() : QStringList()); + + if(fetcher_) { + m_localeCombo->setCurrentData(fetcher_->m_locale); + m_numCast->setValue(fetcher_->m_numCast); + } +} + +void FilmAffinityFetcher::ConfigWidget::saveConfigHook(KConfigGroup& config_) { + config_.writeEntry("Locale", m_localeCombo->currentData().toInt()); + config_.writeEntry("Max Cast", m_numCast->value()); +} + +QString FilmAffinityFetcher::ConfigWidget::preferredName() const { + return FilmAffinityFetcher::defaultName(); +} diff --git a/src/fetch/filmaffinityfetcher.h b/src/fetch/filmaffinityfetcher.h new file mode 100644 index 00000000..d0699912 --- /dev/null +++ b/src/fetch/filmaffinityfetcher.h @@ -0,0 +1,131 @@ +/*************************************************************************** + Copyright (C) 2023 Robby Stephenson <ro...@periapsis.org> + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License or (at your option) version 3 or any later version * + * accepted by the membership of KDE e.V. (or its successor approved * + * by the membership of KDE e.V.), which shall act as a proxy * + * defined in Section 14 of version 3 of the license. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see <http://www.gnu.org/licenses/>. * + * * + ***************************************************************************/ + +#ifndef TELLICO_FETCH_FILMAFFINITYFETCHER_H +#define TELLICO_FETCH_FILMAFFINITYFETCHER_H + +#include "fetcher.h" +#include "configwidget.h" + +#include <QPointer> + +class QSpinBox; +class QUrl; +class KJob; +namespace KIO { + class StoredTransferJob; +} + +namespace Tellico { + + namespace GUI { + class ComboBox; + } + + namespace Fetch { + +/** + * A fetcher for kino-teatr.ua + * + * @author Robby Stephenson + */ +class FilmAffinityFetcher : public Fetcher { +Q_OBJECT + +public: + enum Locale { + ES = 0, + US = 1 + }; + + FilmAffinityFetcher(QObject* parent); + virtual ~FilmAffinityFetcher(); + + virtual QString source() const Q_DECL_OVERRIDE; + virtual bool isSearching() const Q_DECL_OVERRIDE { return m_started; } + virtual bool canSearch(FetchKey k) const Q_DECL_OVERRIDE; + virtual void stop() Q_DECL_OVERRIDE; + virtual Data::EntryPtr fetchEntryHook(uint uid) Q_DECL_OVERRIDE; + virtual Type type() const Q_DECL_OVERRIDE { return FilmAffinity; } + virtual bool canFetch(int type) const Q_DECL_OVERRIDE; + virtual void readConfigHook(const KConfigGroup& config) Q_DECL_OVERRIDE; + + struct LocaleData { + QString siteSlug; + QString tvSeries; + QString year; + QString origTitle; + QString country; + QString runningTime; + QString director; + QString cast; + QString genre; + QString writer; + QString story; + QString producer; + QString distributor; + QString broadcast; + QString music; + QString plot; + }; + static const LocaleData& localeData(int locale); + + virtual Fetch::ConfigWidget* configWidget(QWidget* parent) const Q_DECL_OVERRIDE; + + class ConfigWidget : public Fetch::ConfigWidget { + public: + explicit ConfigWidget(QWidget* parent_, const FilmAffinityFetcher* fetcher = nullptr); + virtual void saveConfigHook(KConfigGroup&) Q_DECL_OVERRIDE; + virtual QString preferredName() const Q_DECL_OVERRIDE; + + private: + GUI::ComboBox* m_localeCombo; + QSpinBox* m_numCast; + }; + friend class ConfigWidget; + + static QString defaultName(); + static QString defaultIcon(); + static StringHash allOptionalFields(); + +private Q_SLOTS: + void slotComplete(KJob* job); + +private: + virtual void search() Q_DECL_OVERRIDE; + virtual FetchRequest updateRequest(Data::EntryPtr entry) Q_DECL_OVERRIDE; + Data::EntryPtr parseEntry(const QString& str); + + QHash<uint, Data::EntryPtr> m_entries; + QHash<uint, QUrl> m_matches; + QPointer<KIO::StoredTransferJob> m_job; + + bool m_started; + Locale m_locale; + int m_numCast; +}; + + } // end namespace +} // end namespace +#endif diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index fdcbd4f9..15542b9c 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -623,6 +623,12 @@ ecm_add_test(externalfetchertest.cpp KF5::XmlGui ) +ecm_add_test(filmaffinityfetchertest.cpp + ../fetch/filmaffinityfetcher.cpp + TEST_NAME filmaffinityfetchertest + LINK_LIBRARIES fetcherstest ${TELLICO_TEST_LIBS} +) + ecm_add_test(filmasterfetchertest.cpp ../fetch/filmasterfetcher.cpp TEST_NAME filmasterfetchertest diff --git a/src/tests/filmaffinityfetchertest.cpp b/src/tests/filmaffinityfetchertest.cpp new file mode 100644 index 00000000..b610002d --- /dev/null +++ b/src/tests/filmaffinityfetchertest.cpp @@ -0,0 +1,173 @@ +/*************************************************************************** + Copyright (C) 2023 Robby Stephenson <ro...@periapsis.org> + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License as * + * published by the Free Software Foundation; either version 2 of * + * the License or (at your option) version 3 or any later version * + * accepted by the membership of KDE e.V. (or its successor approved * + * by the membership of KDE e.V.), which shall act as a proxy * + * defined in Section 14 of version 3 of the license. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program. If not, see <http://www.gnu.org/licenses/>. * + * * + ***************************************************************************/ + +#undef QT_NO_CAST_FROM_ASCII + +#include "filmaffinityfetchertest.h" + +#include "../fetch/filmaffinityfetcher.h" +#include "../entry.h" +#include "../collections/videocollection.h" +#include "../collectionfactory.h" +#include "../images/imagefactory.h" +#include "../fieldformat.h" +#include "../fetch/fetcherjob.h" + +#include <KSharedConfig> + +#include <QTest> + +QTEST_GUILESS_MAIN( FilmAffinityFetcherTest ) + +FilmAffinityFetcherTest::FilmAffinityFetcherTest() : AbstractFetcherTest() { +} + +void FilmAffinityFetcherTest::initTestCase() { + Tellico::ImageFactory::init(); + Tellico::RegisterCollection<Tellico::Data::VideoCollection> registerVideo(Tellico::Data::Collection::Video, "video"); + + m_config = KSharedConfig::openConfig(QString(), KConfig::SimpleConfig)->group(QStringLiteral("filmaffinity")); + m_config.writeEntry("Custom Fields", QStringLiteral("origtitle,filmaffinity")); +} + +void FilmAffinityFetcherTest::testSuperman() { + Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Video, Tellico::Fetch::Title, QStringLiteral("Superman Returns")); + Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::FilmAffinityFetcher(this)); + m_config.writeEntry("Locale", int(Tellico::Fetch::FilmAffinityFetcher::US)); + fetcher->readConfig(m_config); + + Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1); + + QCOMPARE(results.size(), 1); + + // the first entry had better be the right one + Tellico::Data::EntryPtr entry = results.at(0); + + QCOMPARE(entry->field("title"), QStringLiteral("Superman Returns")); + QCOMPARE(entry->field("year"), QStringLiteral("2006")); + QCOMPARE(entry->field("nationality"), QStringLiteral("United States")); + QCOMPARE(entry->field("director"), QStringLiteral("Bryan Singer")); + QCOMPARE(entry->field("writer"), QStringLiteral("Bryan Singer; Michael Dougherty; Dan Harris")); + QCOMPARE(entry->field("composer"), QStringLiteral("John Ottman")); + QCOMPARE(entry->field("studio"), QStringLiteral("Warner Bros.")); + QCOMPARE(set(entry, "genre"), set(QStringLiteral("Sci-Fi; Fantasy; Action; Romance"))); + QCOMPARE(entry->field("running-time"), QStringLiteral("153")); + QStringList castList = Tellico::FieldFormat::splitTable(entry->field(QStringLiteral("cast"))); + QVERIFY(castList.count() > 2); + QCOMPARE(castList.at(0), QStringLiteral("Brandon Routh")); + QCOMPARE(castList.at(1), QStringLiteral("Kevin Spacey")); + QVERIFY(entry->field("plot").startsWith(QStringLiteral("Following a mysterious absence"))); + QVERIFY(!entry->field("cover").isEmpty()); + QVERIFY(!entry->field(QStringLiteral("cover")).contains(QLatin1Char('/'))); + QCOMPARE(entry->field("filmaffinity"), QStringLiteral("https://www.filmaffinity.com/us/film300630.html";)); +} + + +void FilmAffinityFetcherTest::testSupermanES() { + Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Video, Tellico::Fetch::Title, QStringLiteral("Superman Returns")); + Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::FilmAffinityFetcher(this)); + m_config.writeEntry("Locale", int(Tellico::Fetch::FilmAffinityFetcher::ES)); + fetcher->readConfig(m_config); + + Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1); + + QCOMPARE(results.size(), 1); + + // the first entry had better be the right one + Tellico::Data::EntryPtr entry = results.at(0); + + QCOMPARE(entry->field("title"), QStringLiteral("Superman Returns: El regreso")); + QCOMPARE(entry->field("origtitle"), QStringLiteral("Superman Returns")); + QCOMPARE(entry->field("year"), QStringLiteral("2006")); + QCOMPARE(entry->field("nationality"), QStringLiteral("Estados Unidos")); + QCOMPARE(entry->field("director"), QStringLiteral("Bryan Singer")); + QCOMPARE(entry->field("writer"), QStringLiteral("Bryan Singer; Michael Dougherty; Dan Harris")); + QCOMPARE(entry->field("composer"), QStringLiteral("John Ottman")); + QCOMPARE(entry->field("studio"), QStringLiteral("Warner Bros.")); + QCOMPARE(set(entry, "genre"), set(QString::fromUtf8("Ciencia ficción; Fantástico; Acción; Romance"))); + QCOMPARE(entry->field("running-time"), QStringLiteral("153")); + QStringList castList = Tellico::FieldFormat::splitTable(entry->field(QStringLiteral("cast"))); + QVERIFY(castList.count() > 2); + QCOMPARE(castList.at(0), QStringLiteral("Brandon Routh")); + QCOMPARE(castList.at(1), QStringLiteral("Kevin Spacey")); + QVERIFY(entry->field("plot").startsWith(QString::fromUtf8("Tras varios años"))); + QVERIFY(!entry->field("cover").isEmpty()); + QVERIFY(!entry->field(QStringLiteral("cover")).contains(QLatin1Char('/'))); + QCOMPARE(entry->field("filmaffinity"), QStringLiteral("https://www.filmaffinity.com/es/film300630.html";)); +} + +void FilmAffinityFetcherTest::testFirefly() { + Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Video, Tellico::Fetch::Title, QStringLiteral("Firefly 2002")); + Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::FilmAffinityFetcher(this)); + m_config.writeEntry("Locale", int(Tellico::Fetch::FilmAffinityFetcher::US)); + fetcher->readConfig(m_config); + + Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1); + + QCOMPARE(results.size(), 1); + + // the first entry had better be the right one + Tellico::Data::EntryPtr entry = results.at(0); + + QCOMPARE(entry->field("title"), QStringLiteral("Firefly")); + QCOMPARE(entry->field("year"), QStringLiteral("2002")); + QCOMPARE(entry->field("nationality"), QStringLiteral("United States")); + QCOMPARE(entry->field("studio"), QStringLiteral("FOX")); + QVERIFY(!entry->field("plot").isEmpty()); + QVERIFY(!entry->field("cover").isEmpty()); + QVERIFY(!entry->field(QStringLiteral("cover")).contains(QLatin1Char('/'))); + QCOMPARE(entry->field("filmaffinity"), QStringLiteral("https://www.filmaffinity.com/us/film929343.html";)); +} + +void FilmAffinityFetcherTest::testAlcarras() { + Tellico::Fetch::FetchRequest request(Tellico::Data::Collection::Video, Tellico::Fetch::Title, QString::fromUtf8("Alcarràs")); + Tellico::Fetch::Fetcher::Ptr fetcher(new Tellico::Fetch::FilmAffinityFetcher(this)); + m_config.writeEntry("Locale", int(Tellico::Fetch::FilmAffinityFetcher::ES)); + fetcher->readConfig(m_config); + + Tellico::Data::EntryList results = DO_FETCH1(fetcher, request, 1); + + QCOMPARE(results.size(), 1); + + // the first entry had better be the right one + Tellico::Data::EntryPtr entry = results.at(0); + + QCOMPARE(entry->field("title"), QString::fromUtf8("Alcarràs")); + QCOMPARE(entry->field("year"), QStringLiteral("2022")); + QCOMPARE(entry->field("nationality"), QString::fromUtf8("España")); + QCOMPARE(entry->field("director"), QString::fromUtf8("Carla Simón")); + QCOMPARE(entry->field("writer"), QString::fromUtf8("Carla Simón; Arnau Vilaró")); + QCOMPARE(entry->field("composer"), QStringLiteral("Andrea Koch")); + QCOMPARE(entry->field("studio"), QString::fromUtf8("Avalon P.C; Elastica Films; Vilaüt Films; Kino Produzioni; Movistar Plus+; RTVE; TV3")); + QCOMPARE(set(entry, "genre"), set(QStringLiteral("Drama"))); + QCOMPARE(entry->field("running-time"), QStringLiteral("120")); + QStringList castList = Tellico::FieldFormat::splitTable(entry->field(QStringLiteral("cast"))); + QVERIFY(castList.count() > 2); + QCOMPARE(castList.at(0), QStringLiteral("Jordi Pujol Dolcet")); + QCOMPARE(castList.at(1), QStringLiteral("Anna Otín")); + QVERIFY(entry->field("plot").startsWith(QStringLiteral("Durante generaciones"))); + QVERIFY(!entry->field("cover").isEmpty()); + QVERIFY(!entry->field(QStringLiteral("cover")).contains(QLatin1Char('/'))); + QCOMPARE(entry->field("filmaffinity"), QStringLiteral("https://www.filmaffinity.com/es/film457848.html";)); +} diff --git a/src/fetch/fetch.h b/src/tests/filmaffinityfetchertest.h similarity index 55% copy from src/fetch/fetch.h copy to src/tests/filmaffinityfetchertest.h index 4cbbd4bd..2a6a4036 100644 --- a/src/fetch/fetch.h +++ b/src/tests/filmaffinityfetchertest.h @@ -1,5 +1,5 @@ /*************************************************************************** - Copyright (C) 2003-2009 Robby Stephenson <ro...@periapsis.org> + Copyright (C) 2023 Robby Stephenson <ro...@periapsis.org> ***************************************************************************/ /*************************************************************************** @@ -22,98 +22,27 @@ * * ***************************************************************************/ -#ifndef TELLICO_FETCH_H -#define TELLICO_FETCH_H +#ifndef FILMAFFINITYFETCHERTEST_H +#define FILMAFFINITYFETCHERTEST_H -namespace Tellico { - namespace Fetch { +#include "abstractfetchertest.h" -/** - * FetchFirst must be first, and the rest must follow consecutively in value. - * FetchLast must be last! - */ -enum FetchKey { - FetchFirst = 0, - Title, - Person, - ISBN, - UPC, - Keyword, - DOI, - ArxivID, - PubmedID, - LCCN, - Raw, - ExecUpdate, - FetchLast -}; +#include <KConfigGroup> -// real ones must start at 0! -enum Type { - Unknown = -1, - Amazon = 0, - IMDB, - Z3950, - SRU, - Entrez, - ExecExternal, - Yahoo, // Removed - AnimeNfo, // Removed - IBS, - ISBNdb, - GCstarPlugin, - CrossRef, - Citebase, // Removed - Arxiv, - Bibsonomy, - GoogleScholar, - Discogs, - WineCom, - TheMovieDB, - MusicBrainz, - GiantBomb, - OpenLibrary, - Multiple, - Freebase, // Removed - DVDFr, - Filmaster, - Douban, - BiblioShare, - MovieMeter, - GoogleBook, - MAS, // Removed - Springer, - Allocine, - ScreenRush, // Removed - FilmStarts, // Removed - SensaCine, // Removed - Beyazperde, // Removed - HathiTrust, - TheGamesDB, - DBLP, - VNDB, - MRLookup, - BoardGameGeek, - Bedetheque, - OMDB, - KinoPoisk, - VideoGameGeek, - DBC, - IGDB, - Kino, - MobyGames, - ComicVine, - KinoTeatr, - Colnect, - Numista, - TVmaze, - UPCItemDb, - TheTVDB, - RPGGeek, - GamingHistory -}; +class FilmAffinityFetcherTest : public AbstractFetcherTest { +Q_OBJECT +public: + FilmAffinityFetcherTest(); - } -} +private Q_SLOTS: + void initTestCase(); + void testSuperman(); + void testSupermanES(); + void testFirefly(); + void testAlcarras(); + +private: + KConfigGroup m_config; +}; #endif
