Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package python-fanficfare for
openSUSE:Factory checked in at 2023-09-21 22:22:59
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-fanficfare (Old)
and /work/SRC/openSUSE:Factory/.python-fanficfare.new.1770 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-fanficfare"
Thu Sep 21 22:22:59 2023 rev:51 rq:1112734 version:4.27.0
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-fanficfare/python-fanficfare.changes
2023-07-17 19:23:06.941678086 +0200
+++
/work/SRC/openSUSE:Factory/.python-fanficfare.new.1770/python-fanficfare.changes
2023-09-21 22:23:15.061306610 +0200
@@ -1,0 +2,37 @@
+Thu Sep 21 06:13:12 UTC 2023 - Matej Cepl <[email protected]>
+
+- Update to 4.27.0:
+ - AO3: Real fix for adult string change, revert earlier attempted fixes.
+- Update to 4.26.1:
+ - Fix for extratags not being picked up by include_in_subjects
+- Update to 4.26.0:
+ - adapter_royalroadcom: user found a story with no chapters
+ - base_efiction: .string -> stripHTML for nested tags Closes
+ #984
+ - Fix fanfiktionde status parsing #980 - thanks, burny2!
+ - AO3: Make subscribed, markedforlater True/False to match
+ bookmarked/bookmarkprivate/bookmarkrec
+ - Add title replace_metadata(commented) for literotica Ch/Pt
+ titles
+ - AO3: Add subscribed, markedforlater
+ - Filter cookies for flaresolverr
+ - Add _LIST option to
+ include_subject_tags(/extra_subject_tags), refactor
+ - AO3 fix for protected email addr in chapter name breaking
+ chapter datetime
+ - Fix for #979, AO3 Get URLs from Page
+ - adapter_storiesofardacom: Fix for detecting adult question on
+ indiv chapters.
+ - New Site: sunnydaleafterdark.com, thanks inklesspen!
+ - Add anthology_merge_keepsingletocs option, requires new
+ EpubMerge.
+ - Apply mark_new_chapters to new story chapters in
+ Anthologies. #977
+ - Allow mark_new_chapters when 1 chapter in case it changes.
+ - Increase sleep times between cache checks using
+ open_pages_in_browser.
+ - adapter_literotica: 'Fix' clean_chapter_titles for titles
+ ending with Pt or Ch
+ - Fix for ficbook.net date change. Closes #973
+
+-------------------------------------------------------------------
Old:
----
FanFicFare-4.25.0.tar.gz
New:
----
FanFicFare-4.27.0.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-fanficfare.spec ++++++
--- /var/tmp/diff_new_pack.xRApFd/_old 2023-09-21 22:23:16.645364091 +0200
+++ /var/tmp/diff_new_pack.xRApFd/_new 2023-09-21 22:23:16.649364236 +0200
@@ -20,7 +20,7 @@
%define modnamedown fanficfare
%define skip_python2 1
Name: python-fanficfare
-Version: 4.25.0
+Version: 4.27.0
Release: 0
Summary: Tool for making eBooks from stories on fanfiction and other
web sites
License: GPL-3.0-only
++++++ FanFicFare-4.25.0.tar.gz -> FanFicFare-4.27.0.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/FanFicFare-4.25.0/calibre-plugin/__init__.py
new/FanFicFare-4.27.0/calibre-plugin/__init__.py
--- old/FanFicFare-4.25.0/calibre-plugin/__init__.py 2023-07-03
22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/calibre-plugin/__init__.py 2023-09-01
15:30:19.000000000 +0200
@@ -33,7 +33,7 @@
from calibre.customize import InterfaceActionBase
# pulled out from FanFicFareBase for saving in prefs.py
-__version__ = (4, 25, 0)
+__version__ = (4, 27, 0)
## Apparently the name for this class doesn't matter--it was still
## 'demo' for the first few versions.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/FanFicFare-4.25.0/calibre-plugin/fff_plugin.py
new/FanFicFare-4.27.0/calibre-plugin/fff_plugin.py
--- old/FanFicFare-4.25.0/calibre-plugin/fff_plugin.py 2023-07-03
22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/calibre-plugin/fff_plugin.py 2023-09-01
15:30:19.000000000 +0200
@@ -2069,13 +2069,25 @@
coverpath = calcoverpath
# logger.debug('coverpath:%s'%coverpath)
- self.get_epubmerge_plugin().do_merge(tmp.name,
- [ x['outfile'] for x in
good_list ],
- tags=mergebook['tags'],
- titleopt=mergebook['title'],
- keepmetadatafiles=True,
- source=mergebook['url'],
- coverjpgpath=coverpath)
+ mrg_args = [tmp.name,
+ [ x['outfile'] for x in good_list ],]
+ mrg_kwargs = {
+ 'tags':mergebook['tags'],
+ 'titleopt':mergebook['title'],
+ 'keepmetadatafiles':True,
+ 'source':mergebook['url'],
+ 'coverjpgpath':coverpath
+ }
+ logger.debug('anthology_merge_keepsingletocs:%s'%
+ mergebook['anthology_merge_keepsingletocs'])
+ if mergebook['anthology_merge_keepsingletocs']:
+ if
self.get_epubmerge_plugin().interface_action_base_plugin.version >= (2,15,3):
+
mrg_kwargs['keepsingletocs']=mergebook['anthology_merge_keepsingletocs']
+ else:
+ logger.warn("anthology_merge_keepsingletocs:true
ignored--your EpubMerge doesn't support it yet.")
+
+ self.get_epubmerge_plugin().do_merge(*mrg_args,**mrg_kwargs)
+
mergebook['collision'] = options['collision'] = OVERWRITEALWAYS
errorcol_label = self.get_custom_col_label(prefs['errorcol'])
@@ -2955,7 +2967,7 @@
book['comments'] += '</div>'
# logger.debug(book['comments'])
- configuration = get_fff_config(book['url'],options['fileform'])
+ configuration =
get_fff_config(options.get('anthology_url',''),options['fileform'])
if existingbook:
book['title'] = deftitle = existingbook['title']
if prefs['anth_comments_newonly']:
@@ -3022,6 +3034,10 @@
if 'anthology_url' in options:
book['url'] = options['anthology_url']
+ # cheesy, but easier than getting hold of configuration again
+ # later.
+ book['anthology_merge_keepsingletocs'] =
configuration.getConfig('anthology_merge_keepsingletocs',False)
+
return book
def split_text_to_urls(urls):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/FanFicFare-4.25.0/calibre-plugin/jobs.py
new/FanFicFare-4.27.0/calibre-plugin/jobs.py
--- old/FanFicFare-4.25.0/calibre-plugin/jobs.py 2023-07-03
22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/calibre-plugin/jobs.py 2023-09-01
15:30:19.000000000 +0200
@@ -251,6 +251,17 @@
if not story.getMetadata("series") and 'calibre_series' in book:
adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
+ # logger.debug(merge)
+ # logger.debug(book.get('epub_for_update','(NONE)'))
+ # logger.debug(options.get('mergebook','(NOMERGEBOOK)'))
+
+ # is a merge, is a pre-existing anthology, and is not a
pre-existing book in anthology.
+ if merge and 'mergebook' in options and 'epub_for_update' not in
book:
+ # internal for plugin anthologies to mark chapters
+ # (new) in new stories
+ story.setMetadata("newforanthology","true")
+ logger.debug("metadata
newforanthology:%s"%story.getMetadata("newforanthology"))
+
# set PI version instead of default.
if 'version' in options:
story.setMetadata('version',options['version'])
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/FanFicFare-4.25.0/calibre-plugin/plugin-defaults.ini
new/FanFicFare-4.27.0/calibre-plugin/plugin-defaults.ini
--- old/FanFicFare-4.25.0/calibre-plugin/plugin-defaults.ini 2023-07-03
22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/calibre-plugin/plugin-defaults.ini 2023-09-01
15:30:19.000000000 +0200
@@ -153,8 +153,19 @@
## number, or one of _. []()&'-
output_filename_safepattern:(^\.|/\.|[^a-zA-Z0-9_\. \[\]\(\)&'-]+)
-## entries to make epub subjects and calibre tags
+## include_subject_tags: entries to make epub subjects and calibre tags
## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last
Update: %Y/%m/%d"
+##
+## By default, entries listed in include_subject_tags are used as
+## lists, skipping any <entry>_LIST processing in
+## replace_metadata/etc. You can instead use <entry>_LIST in
+## include_subject_tags to get the whole list as one string.
+##
+## You can also use <entry>.SPLIT to split each value in the entry on
+## ',' for individual values. .SPLIT was basically obsoleted by \,
+## splitting, but may still be used in some users' config. Can also
+## be combined with _LIST as <entry>_LIST.SPLIT to get both
+## <entry>_LIST processing in replace_metadata/etc *and* split by ','
include_subject_tags: extratags, genre, category, characters, ships, status
## extra tags (comma separated) to include, primarily for epub.
@@ -471,9 +482,25 @@
## If set to latestonly, only new chapters downloaded this time will
## be marked (new) and existing chapters will have any (new) marks
## removed.
+##
## mark_new_chapters can be true, false or latestonly
mark_new_chapters:false
+## (Plugin Only) - When mark_new_chapters:true, new stories added to
+## an existing anthology will have *all* chapters marked (new). If
+## set to latestonly, existing (new) marks will only be removed when
+## that particular story within the anthology is updated--either
+## because it has new chapter(s) or you are using Update Always.
+##
+## By default, EpubMerge only creates one TOC entry for a story that
+## has only one chapter (not counting titlepage and logpage). Because
+## that TOC entry is the story title, the *chapter* title doesn't
+## appear in the anthology TOC. Setting
+## anthology_merge_keepsingletocs:true will keep the full TOC for all
+## anthology stories, even when only one chapter. Helpful to see
+## (new) marks in TOC when mark_new_chapters:true
+#anthology_merge_keepsingletocs:false
+
## chapter title patterns use python template substitution. The
## ${number} is the 'chapter' number and ${title} is the chapter
## title, after applying chapter_title_strip_pattern. ${index04} is
@@ -791,9 +818,6 @@
include_in_genre:genre,forumtags
include_in_category:category,tagsfromtitle,parentforums
-## Previously was:
-#add_to_include_subject_tags:,tagsfromtitle.SPLIT,forumtags
-
## base_xenforoforum reads Published and Updated datetimes from
## Threadmarks if used, or from the posted & updated times of the
## 'first' post if no threadmarks.
@@ -1381,7 +1405,7 @@
## slow_down_sleep_time now applied to test1 chapters (with 0.5-1.5
## random fudge)
#slow_down_sleep_time:2
-extratags: FanFiction,Testing
+#extratags: FanFiction,Testing
# extracategories:Fafner
# extragenres:Romance,Fluff
@@ -1409,19 +1433,6 @@
# .*-Centered=>
# characters=>Harry P\.=>Harry Potter
-
-[test1.com:txt]
-
-## If necessary, you can define [<site>:<format>] sections to
-## customize the formats differently for the same site. Overrides
-## defaults, format and site.
-
-extratags: FanFiction,Testing,Text
-
-[test1.com:html]
-
-extratags: FanFiction,Testing,HTML
-
[teststory:defaults]
valid_entries:title,author_list,authorId_list,authorUrl_list,storyUrl,
@@ -1549,9 +1560,10 @@
extra_valid_entries:fandoms, freeformtags, freefromtags,
ao3categories, comments, chapterslashtotal, chapterstotal, kudos,
hits, bookmarks, collections, byline, bookmarked, bookmarktags,
- bookmarksummary, bookmarkprivate, bookmarkrec, restricted, series00,
- series01, series02, series03, series00Url, series01Url, series02Url,
- series03Url, series00HTML, series01HTML, series02HTML, series03HTML
+ bookmarksummary, bookmarkprivate, bookmarkrec, subscribed,
+ markedforlater, restricted, series00, series01, series02, series03,
+ series00Url, series01Url, series02Url, series03Url, series00HTML,
+ series01HTML, series02HTML, series03HTML
fandoms_label:Fandoms
freeformtags_label:Freeform Tags
freefromtags_label:Freeform Tags
@@ -1571,6 +1583,8 @@
bookmarksummary_label:My Bookmark Summary
bookmarkprivate_label:My Bookmark Private
bookmarkrec_label:My Bookmark Rec
+markedforlater_label:I Marked for Later
+subscribed_label:I Subscribed
restricted_label:Restricted to Registered Users
series00HTML_label:Series
series01HTML_label:Additional Series
@@ -2167,6 +2181,12 @@
## or just use the text. If this can't be done, the full title is used.
clean_chapter_titles: false
+## Some stories mistakenly include 'Ch' or 'Pt' at the end of the
+## story title. Appears to be a site bug or common author error. Copy
+## these to your personal.ini (and uncomment) to correct.
+#add_to_replace_metadata:
+# title=> (Ch|Pt)$=>
+
## Add the chapter description at the start of each chapter.
description_in_chapter: false
@@ -2503,9 +2523,10 @@
extra_valid_entries:fandoms, freeformtags, freefromtags,
ao3categories, comments, chapterslashtotal, chapterstotal, kudos,
hits, bookmarks, collections, byline, bookmarked, bookmarktags,
- bookmarksummary, bookmarkprivate, bookmarkrec, restricted, series00,
- series01, series02, series03, series00Url, series01Url, series02Url,
- series03Url, series00HTML, series01HTML, series02HTML, series03HTML
+ bookmarksummary, bookmarkprivate, bookmarkrec, subscribed,
+ markedforlater, restricted, series00, series01, series02, series03,
+ series00Url, series01Url, series02Url, series03Url, series00HTML,
+ series01HTML, series02HTML, series03HTML
fandoms_label:Fandoms
freeformtags_label:Freeform Tags
freefromtags_label:Freeform Tags
@@ -2525,6 +2546,8 @@
bookmarksummary_label:My Bookmark Summary
bookmarkprivate_label:My Bookmark Private
bookmarkrec_label:My Bookmark Rec
+markedforlater_label:I Marked for Later
+subscribed_label:I Subscribed
restricted_label:Restricted to Registered Users
series00HTML_label:Series
series01HTML_label:Additional Series
@@ -3554,7 +3577,10 @@
[www.storiesofarda.com]
## Some sites do not require a login, but do require the user to
## confirm they are adult for adult content. In commandline version,
-## this should go in your personal.ini, not defaults.ini.
+## this should go in your personal.ini, not defaults.ini. This is
+## more important for storiesofarda.com than most sites.
+## storiesofarda.com puts 'are you adult' question in front of
+## individual chapters, not stories.
#is_adult:true
## Site dedicated to these categories/characters/ships
@@ -3562,6 +3588,24 @@
website_encodings:Windows-1252,utf8
+[www.sunnydaleafterdark.com]
+## Site dedicated to these categories/characters/ships
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config. In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content. In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+extracategories:Buffy: The Vampire Slayer
+extracharacters:Buffy, Spike
+extraships:Spike/Buffy
+
[www.swi.org.ru]
use_basic_cache:true
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/FanFicFare-4.25.0/calibre-plugin/translations/ru.po
new/FanFicFare-4.27.0/calibre-plugin/translations/ru.po
--- old/FanFicFare-4.25.0/calibre-plugin/translations/ru.po 2023-07-03
22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/calibre-plugin/translations/ru.po 2023-09-01
15:30:19.000000000 +0200
@@ -6,13 +6,13 @@
# 3f8f9c00d81c4c6472ea90c31062787f_3a6a5f8
<579c3b116bb0c0395bb3df6f1ae34246_210356>, 2014
# Vikarti Anatra <[email protected]>, 2020
# ÐлекÑей ÐоÑобÑÑв <[email protected]>, 2019
-# ÐикÑоÑ, 2022
+# Victor, 2022
msgid ""
msgstr ""
"Project-Id-Version: calibre-plugins\n"
"POT-Creation-Date: 2022-07-06 11:14-0500\n"
"PO-Revision-Date: 2014-06-19 22:55+0000\n"
-"Last-Translator: ÐикÑоÑ, 2022\n"
+"Last-Translator: Victor, 2022\n"
"Language-Team: Russian
(http://app.transifex.com/calibre/calibre-plugins/language/ru/)\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/FanFicFare-4.25.0/fanficfare/adapters/__init__.py
new/FanFicFare-4.27.0/fanficfare/adapters/__init__.py
--- old/FanFicFare-4.25.0/fanficfare/adapters/__init__.py 2023-07-03
22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/adapters/__init__.py 2023-09-01
15:30:19.000000000 +0200
@@ -145,6 +145,7 @@
from . import adapter_deviantartcom
from . import adapter_merengohu
from . import adapter_readonlymindcom
+from . import adapter_wwwsunnydaleafterdarkcom
## This bit of complexity allows adapters to be added by just adding
## importing. It eliminates the long if/else clauses we used to need
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/FanFicFare-4.25.0/fanficfare/adapters/adapter_archiveofourownorg.py
new/FanFicFare-4.27.0/fanficfare/adapters/adapter_archiveofourownorg.py
--- old/FanFicFare-4.25.0/fanficfare/adapters/adapter_archiveofourownorg.py
2023-07-03 22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/adapters/adapter_archiveofourownorg.py
2023-09-01 15:30:19.000000000 +0200
@@ -174,7 +174,11 @@
if 'This work is part of an ongoing challenge and will be revealed
soon!' in meta:
raise exceptions.FailedToDownload('Site says: "This work is part
of an ongoing challenge and will be revealed soon!"')
- if "This work could have adult content. If you proceed you have agreed
that you are willing to see such content." in meta:
+ if '<p class="caution">' in meta:
+ logger.debug('<p class="caution"> found. If download fails, check
for changed "is adult" string')
+ # This work could have adult content. If you continue,
you have agreed that you are willing to see such content.
+ # This work could have adult content. If you proceed you
have agreed that you are willing to see such content.
+ if re.search(r"This work could have adult content. If you
(continue,|proceed) you have agreed that you are willing to see such content.",
meta):
if self.addurl:
## "?view_adult=true" doesn't work on base story
## URL anymore, which means we have to
@@ -216,19 +220,28 @@
# deliberately using always_login instead of checking for
# actual login so we don't have a case where these show up
# for a user only when they get user-restricted stories.
- try:
- # is bookmarked if has update /bookmarks/ form --
- # create bookmark form uses different url
- self.story.setMetadata('bookmarked',
- None !=
metasoup.find('form',action=re.compile(r'^/bookmarks/')))
+
+ # is bookmarked if has update /bookmarks/ form --
+ # create bookmark form uses different url
+ self.story.setMetadata('bookmarked',
+ None !=
metasoup.find('form',action=re.compile(r'^/bookmarks/')))
+ if
metasoup.find('input',id='bookmark_tag_string').has_attr('value'):
self.story.extendList('bookmarktags',
metasoup.find('input',id='bookmark_tag_string')['value'].split(', '))
- self.story.setMetadata('bookmarkprivate',
-
metasoup.find('input',id='bookmark_private').has_attr('checked'))
- self.story.setMetadata('bookmarkrec',
-
metasoup.find('input',id='bookmark_rec').has_attr('checked'))
- except KeyError:
- pass
+ self.story.setMetadata('bookmarkprivate',
+
metasoup.find('input',id='bookmark_private').has_attr('checked'))
+ self.story.setMetadata('bookmarkrec',
+
metasoup.find('input',id='bookmark_rec').has_attr('checked'))
+
+ # detect subscription by unsub button
+ # logger.debug(metasoup.find('input',value="Unsubscribe"))
+ self.story.setMetadata('subscribed',
+ metasoup.find('input',value="Unsubscribe")
is not None)
+ # detect 'marked for later' by 'Mark as Read' button
+ # logger.debug(metasoup.find('a',
href=re.compile(r'/mark_as_read$')))
+ self.story.setMetadata('markedforlater',
+ metasoup.find('a',
href=re.compile(r'/mark_as_read$')) is not None)
+
self.story.setMetadata('bookmarksummary',
stripHTML(metasoup.find('textarea',id='bookmark_notes')))
@@ -278,7 +291,7 @@
for index, chapter in enumerate(chapters):
# strip just in case there's tags, like <i> in chapter titles.
# (2013-09-21)
- date = stripHTML(chapter.findNext('span'))[1:-1]
+ date =
stripHTML(chapter.findNext('span',class_='datetime'))[1:-1]
chapterDate = makeDate(date,self.dateformat)
self.add_chapter(chapter,'https://'+self.host+chapter['href'],
{'date':chapterDate.strftime(self.getConfig("datechapter_format",self.getConfig("datePublished_format","%Y-%m-%d")))})
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/FanFicFare-4.25.0/fanficfare/adapters/adapter_fanfiktionde.py
new/FanFicFare-4.27.0/fanficfare/adapters/adapter_fanfiktionde.py
--- old/FanFicFare-4.25.0/fanficfare/adapters/adapter_fanfiktionde.py
2023-07-03 22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/adapters/adapter_fanfiktionde.py
2023-09-01 15:30:19.000000000 +0200
@@ -163,11 +163,11 @@
except e:
logger.debug("Failed to find native status:%s"%e)
- if head.find('span',title='Fertiggestellt'):
+ if head.find('span',title='fertiggestellt'):
self.story.setMetadata('status', 'Completed')
- elif head.find('span',title='Pausiert'):
+ elif head.find('span',title='pausiert'):
self.story.setMetadata('status', 'Paused')
- elif head.find('span',title='Abgebrochen'):
+ elif head.find('span',title='abgebrochen'):
self.story.setMetadata('status', 'Cancelled')
else:
self.story.setMetadata('status', 'In-Progress')
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/FanFicFare-4.25.0/fanficfare/adapters/adapter_ficbooknet.py
new/FanFicFare-4.27.0/fanficfare/adapters/adapter_ficbooknet.py
--- old/FanFicFare-4.25.0/fanficfare/adapters/adapter_ficbooknet.py
2023-07-03 22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/adapters/adapter_ficbooknet.py
2023-09-01 15:30:19.000000000 +0200
@@ -152,6 +152,9 @@
if name in update:
update = update.replace(name,num)
+ ## remove extra ' г.' on date.
+ update = update.replace(' г.','')
+ pubdate = pubdate.replace(' г.','')
self.story.setMetadata('dateUpdated', makeDate(update,
self.dateformat))
self.story.setMetadata('datePublished', makeDate(pubdate,
self.dateformat))
self.story.setMetadata('language','Russian')
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/FanFicFare-4.25.0/fanficfare/adapters/adapter_literotica.py
new/FanFicFare-4.27.0/fanficfare/adapters/adapter_literotica.py
--- old/FanFicFare-4.25.0/fanficfare/adapters/adapter_literotica.py
2023-07-03 22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/adapters/adapter_literotica.py
2023-09-01 15:30:19.000000000 +0200
@@ -163,7 +163,7 @@
## site is now using language specific german.lit... etc on author
pages.
## site is now back to using www.lit... etc on author pages.
search_url_re = r"https?://"+LANG_RE+r"(\.i)?\." +
re.escape(self.getSiteDomain()) + self.url[self.url.index('/s/'):]+r"$"
- logger.debug(search_url_re)
+ # logger.debug(search_url_re)
storyLink = soupAuth.find('a', href=re.compile(search_url_re))
# storyLink = soupAuth.find('a',
href=re.compile(r'.*literotica.com/s/'+re.escape(self.story.getMetadata('storyId'))
))
# storyLink = soupAuth.find('a',
href=re.compile(r'(https?:)?'+re.escape(self.url[self.url.index(':')+1:]).replace(r'www',r'[^\.]+')
))
@@ -234,14 +234,18 @@
chapter_title = chapterLink.text
if self.getConfig("clean_chapter_titles"):
# logger.debug('\tChapter Name: "%s"' % chapterLink.text)
- if
chapterLink.text.lower().startswith(seriesTitle.lower()):
+ seriesTitle = seriesTitle.lower()
+ # strip trailing ch or pt before doing the chapter clean.
+ # doesn't remove from story title metadata
+ seriesTitle = re.sub(r'^(.*?)(
(ch|pt))?$',r'\1',seriesTitle)
+ if chapterLink.text.lower().startswith(seriesTitle):
chapter = chapterLink.text[len(seriesTitle):].strip()
# logger.debug('\tChapter: "%s"' % chapter)
if chapter == '':
chapter_title = 'Chapter %d' %
(self.num_chapters() + 1)
# Sometimes the first chapter does not have type
of chapter
if self.num_chapters() == 0:
- logger.debug('\tChapter: first chapter without
chapter type')
+ # logger.debug('\tChapter: first chapter
without chapter type')
chapter_name_type = None
else:
separater_char = chapter[0]
@@ -255,18 +259,18 @@
except:
chapter_title = 'Chapter %s' % chapter
chapter_name_type = 'Chapter' if
chapter_name_type is None else chapter_name_type
- logger.debug('\tChapter:
chapter_name_type="%s"' % chapter_name_type)
+ # logger.debug('\tChapter:
chapter_name_type="%s"' % chapter_name_type)
elif chapter.lower().startswith('pt.'):
- chapter = chapter[len('pt.'):]
+ chapter = chapter[len('pt.'):].strip()
try:
chapter_title = 'Part %d' % int(chapter)
except:
chapter_title = 'Part %s' % chapter
chapter_name_type = 'Part' if
chapter_name_type is None else chapter_name_type
- logger.debug('\tChapter:
chapter_name_type="%s"' % chapter_name_type)
+ # logger.debug('\tChapter:
chapter_name_type="%s"' % chapter_name_type)
elif separater_char in [":", "-"]:
chapter_title = chapter
- logger.debug('\tChapter: taking chapter text
as whole')
+ # logger.debug('\tChapter: taking chapter text
as whole')
# pages include full URLs.
chapurl = chapterLink['href']
@@ -288,9 +292,9 @@
if self.getConfig("clean_chapter_titles") \
and chapter_name_type is not None \
and not chapters[0][0].startswith(chapter_name_type):
- logger.debug('\tChapter: chapter_name_type="%s"' %
chapter_name_type)
- logger.debug('\tChapter: first chapter="%s"' % chapters[0][0])
- logger.debug('\tChapter: first chapter number="%s"' %
chapters[0][0][len('Chapter'):])
+ # logger.debug('\tChapter: chapter_name_type="%s"' %
chapter_name_type)
+ # logger.debug('\tChapter: first chapter="%s"' %
chapters[0][0])
+ # logger.debug('\tChapter: first chapter number="%s"' %
chapters[0][0][len('Chapter'):])
chapters[0] = ("%s %s" % (chapter_name_type,
chapters[0][0][len('Chapter'):].strip()),
chapters[0][1],
chapters[0][2],
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/FanFicFare-4.25.0/fanficfare/adapters/adapter_royalroadcom.py
new/FanFicFare-4.27.0/fanficfare/adapters/adapter_royalroadcom.py
--- old/FanFicFare-4.25.0/fanficfare/adapters/adapter_royalroadcom.py
2023-07-03 22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/adapters/adapter_royalroadcom.py
2023-09-01 15:30:19.000000000 +0200
@@ -178,6 +178,11 @@
chapters = soup.find('table',{'id':'chapters'}).find('tbody')
tds = [tr.findAll('td') for tr in chapters.findAll('tr')]
+
+ if not tds:
+ raise exceptions.FailedToDownload(
+ "Story has no chapters: %s" % url)
+
# Links in the RR ToC page are in the normalized long form, so match
is simpler than in normalize_chapterurl()
chap_pattern_long =
r"https?://(?:www\.)?royalroadl?\.com/fiction/\d+/[^/]+/chapter/(\d+)/[^/]+/?$"
for chapter,date in tds:
@@ -258,7 +263,7 @@
div = soup.find('div',{'class':"chapter-inner chapter-content"})
# TODO: these stories often have tables in, but these wont render
correctly
- # defaults.ini output CSS now outlines/pads the tables, at least.
+ # defaults.ini output CSS now outlines/pads the tables, at least.
if None == div:
raise exceptions.FailedToDownload("Error downloading Chapter: %s!
Missing required element!" % url)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/FanFicFare-4.25.0/fanficfare/adapters/adapter_storiesofardacom.py
new/FanFicFare-4.27.0/fanficfare/adapters/adapter_storiesofardacom.py
--- old/FanFicFare-4.25.0/fanficfare/adapters/adapter_storiesofardacom.py
2023-07-03 22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/adapters/adapter_storiesofardacom.py
2023-09-01 15:30:19.000000000 +0200
@@ -134,14 +134,14 @@
else:
data = self.get_request(url)
+ if "Please indicate that you are an adult by selecting the appropriate
choice below" in data:
+ raise exceptions.FailedToDownload("Chapter requires you be an
adult. Set is_adult in personal.ini (chapter url:%s)" % url)
+
data = data[data.index('<table width="90%" align="center">'):]
data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
soup = self.make_soup(data)
- if "Please indicate that you are an adult by selecting the appropriate
choice below" in data:
- raise exceptions.FailedToDownload("Chapter requires you be an
adult. Set is_adult in personal.ini (chapter url:%s)" % url)
-
div = soup.find('table', {'width' : '90%'}).find('td')
div.name='div'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/FanFicFare-4.25.0/fanficfare/adapters/adapter_wwwsunnydaleafterdarkcom.py
new/FanFicFare-4.27.0/fanficfare/adapters/adapter_wwwsunnydaleafterdarkcom.py
---
old/FanFicFare-4.25.0/fanficfare/adapters/adapter_wwwsunnydaleafterdarkcom.py
1970-01-01 01:00:00.000000000 +0100
+++
new/FanFicFare-4.27.0/fanficfare/adapters/adapter_wwwsunnydaleafterdarkcom.py
2023-09-01 15:30:19.000000000 +0200
@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import absolute_import
+from ..htmlcleanup import stripHTML
+
+# Software: eFiction
+from .base_efiction_adapter import BaseEfictionAdapter
+
+class WWWSunnydaleAfterDarkComAdapter(BaseEfictionAdapter):
+
+ @classmethod
+ def getProtocol(self):
+ """
+ Some, but not all site now require https.
+ """
+ return "https"
+
+ @staticmethod
+ def getSiteDomain():
+ return 'www.sunnydaleafterdark.com'
+
+ @classmethod
+ def getSiteAbbrev(self):
+ return 'sad'
+
+ @classmethod
+ def getDateFormat(self):
+ return r"%m/%d/%y"
+
+ def extractChapterUrlsAndMetadata(self):
+ ## Call super of extractChapterUrlsAndMetadata().
+ ## base_efiction leaves the soup in self.html.
+ super(WWWSunnydaleAfterDarkComAdapter,
self).extractChapterUrlsAndMetadata()
+
+ ## attempt to fetch rating from title line:
+ ## "Do You Think This Is Love? by Supernatural Beings [PG]"
+ r = stripHTML(self.html.find("div", {"id": "pagetitle"}))
+ if '[' in r and ']' in r:
+ self.story.setMetadata('rating',
+ r[r.index('[')+1:r.index(']')])
+
+ def make_soup(self, data):
+ soup = super(WWWSunnydaleAfterDarkComAdapter, self).make_soup(data)
+ ## This site uses Javascript to "hide" warnings, for spoiler reasons
+ ## <span class="label">Warnings: <span class="revealSpoiler"
onclick="this.getElementsByClassName('spoiler')[0].classList.remove('spoiler');">(Click
Here To Reveal) <span class="spoiler">Warning A, Warning B, Warning Y, Warning
Z</span></span></span>
+ ## We need to remove the revealSpoiler spans and replace them with the
contents
+ ## of the enclosed spoiler spans.
+ infobox = soup.find("div", "infobox")
+ if infobox is not None:
+ for revealSpoiler in infobox.find_all("span",
class_="revealSpoiler"):
+ parent = revealSpoiler.parent
+ spoiler = revealSpoiler.find("span", class_="spoiler")
+ spoiler.extract()
+ revealSpoiler.replace_with(spoiler)
+ spoiler.unwrap()
+ parent.smooth()
+ return soup
+
+ def handleMetadataPair(self, key, value):
+ ## Inexplicably puts the entire Genres string inside the label span
+ ## Likewise Warnings, which also have the spoiler javascript (removed
in make_soup)
+ if key.startswith("Genre") or key.startswith("Warning"):
+ key, value = key.split(': ')
+ super(WWWSunnydaleAfterDarkComAdapter, self).handleMetadataPair(key,
value)
+
+
+def getClass():
+ return WWWSunnydaleAfterDarkComAdapter
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/FanFicFare-4.25.0/fanficfare/adapters/base_adapter.py
new/FanFicFare-4.27.0/fanficfare/adapters/base_adapter.py
--- old/FanFicFare-4.25.0/fanficfare/adapters/base_adapter.py 2023-07-03
22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/adapters/base_adapter.py 2023-09-01
15:30:19.000000000 +0200
@@ -459,7 +459,7 @@
self.before_get_urls_from_page(url,normalize)
# this way it uses User-Agent or other special settings.
- data = self.get_request(url,usecache=True)
+ data = self.get_request(url,usecache=False)
series = self.get_series_from_page(url,data,normalize)
if series:
# just to make it easier for adapters.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/FanFicFare-4.25.0/fanficfare/adapters/base_efiction_adapter.py
new/FanFicFare-4.27.0/fanficfare/adapters/base_efiction_adapter.py
--- old/FanFicFare-4.25.0/fanficfare/adapters/base_efiction_adapter.py
2023-07-03 22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/adapters/base_efiction_adapter.py
2023-09-01 15:30:19.000000000 +0200
@@ -409,9 +409,9 @@
pagetitleDiv = soup.find("div", {"id": "pagetitle"})
if pagetitleDiv.find('a') is None:
raise exceptions.FailedToDownload("Couldn't find title and author")
- self.story.setMetadata('title', pagetitleDiv.find("a").string)
+ self.story.setMetadata('title', stripHTML(pagetitleDiv.find("a")))
authorLink = pagetitleDiv.findAll("a")[1]
- self.story.setMetadata('author', authorLink.string)
+ self.story.setMetadata('author', stripHTML(authorLink))
self.story.setMetadata('authorId', re.search(r"\d+",
authorLink['href']).group(0))
self.story.setMetadata('authorUrl',
self.getViewUserUrl(self.story.getMetadata('authorId')))
@@ -431,7 +431,7 @@
else:
valueStr += unicode(nextEl)
nextEl = nextEl.nextSibling
- key = labelSpan.string.strip()
+ key = stripHTML(labelSpan)
## strip trailing colons
key = re.sub(r"\s*:\s*$", "", key)
@@ -467,8 +467,8 @@
## Had a problem with an author putting <b>0.</b> in the text.
if chapterLink and chapterLink['href'].startswith('#'):
chapterLink['href'] = "%s&chapter=%s" % (self.url,
chapterId)
- if chapterLink.string != self.getBacktoIndex():
- self.add_chapter(chapterLink.string,
chapterLink['href'])
+ if stripHTML(chapterLink) != self.getBacktoIndex():
+ self.add_chapter(stripHTML(chapterLink),
chapterLink['href'])
## Store reference to soup for getChapterText
self.html = soup
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/FanFicFare-4.25.0/fanficfare/cli.py
new/FanFicFare-4.27.0/fanficfare/cli.py
--- old/FanFicFare-4.25.0/fanficfare/cli.py 2023-07-03 22:38:14.000000000
+0200
+++ new/FanFicFare-4.27.0/fanficfare/cli.py 2023-09-01 15:30:19.000000000
+0200
@@ -28,7 +28,7 @@
import os, sys, platform
-version="4.25.0"
+version="4.27.0"
os.environ['CURRENT_VERSION_ID']=version
global_cache = 'global_cache'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/FanFicFare-4.25.0/fanficfare/configurable.py
new/FanFicFare-4.27.0/fanficfare/configurable.py
--- old/FanFicFare-4.25.0/fanficfare/configurable.py 2023-07-03
22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/configurable.py 2023-09-01
15:30:19.000000000 +0200
@@ -342,6 +342,9 @@
'langcode',
'output_css',
'cover_image',
+ 'newforanthology' # internal for plugin anthologies
+ # to mark chapters (new) in new
+ # stories
])
def get_valid_entries():
@@ -363,6 +366,7 @@
'always_overwrite',
'anthology_tags',
'anthology_title_pattern',
+ 'anthology_merge_keepsingletocs',
'background_color',
'bulk_load',
'chapter_end',
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/FanFicFare-4.25.0/fanficfare/defaults.ini
new/FanFicFare-4.27.0/fanficfare/defaults.ini
--- old/FanFicFare-4.25.0/fanficfare/defaults.ini 2023-07-03
22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/defaults.ini 2023-09-01
15:30:19.000000000 +0200
@@ -173,8 +173,19 @@
## number, or one of _. []()&'-
output_filename_safepattern:(^\.|/\.|[^a-zA-Z0-9_\. \[\]\(\)&'-]+)
-## entries to make epub subjects and calibre tags
+## include_subject_tags: entries to make epub subjects and calibre tags
## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last
Update: %Y/%m/%d"
+##
+## By default, entries listed in include_subject_tags are used as
+## lists, skipping any <entry>_LIST processing in
+## replace_metadata/etc. You can instead use <entry>_LIST in
+## include_subject_tags to get the whole list as one string.
+##
+## You can also use <entry>.SPLIT to split each value in the entry on
+## ',' for individual values. .SPLIT was basically obsoleted by \,
+## splitting, but may still be used in some users' config. Can also
+## be combined with _LIST as <entry>_LIST.SPLIT to get both
+## <entry>_LIST processing in replace_metadata/etc *and* split by ','
include_subject_tags: extratags, genre, category, characters, ships,
lastupdate, status
## extra tags (comma separated) to include, primarily for epub.
@@ -456,6 +467,7 @@
## If set to latestonly, only new chapters downloaded this time will
## be marked (new) and existing chapters will have any (new) marks
## removed.
+##
## mark_new_chapters can be true, false or latestonly
mark_new_chapters:false
@@ -791,9 +803,6 @@
include_in_genre:genre,forumtags
include_in_category:category,tagsfromtitle,parentforums
-## Previously was:
-#add_to_include_subject_tags:,tagsfromtitle.SPLIT,forumtags
-
## base_xenforoforum reads Published and Updated datetimes from
## Threadmarks if used, or from the posted & updated times of the
## 'first' post if no threadmarks.
@@ -1150,7 +1159,6 @@
## include images from img tags in the body and summary of
## stories. Images will be converted to jpg for size if possible.
## include_images is *only* available in epub and html output format.
-## include_images is *not* available in the web service in any format.
#include_images:false
## Quality level to use when converting images to jpg. Range is 0-100,
@@ -1385,7 +1393,7 @@
## slow_down_sleep_time now applied to test1 chapters (with 0.5-1.5
## random fudge)
#slow_down_sleep_time:2
-extratags: FanFiction,Testing
+#extratags: FanFiction,Testing
# extracategories:Fafner
# extragenres:Romance,Fluff
@@ -1413,19 +1421,6 @@
# .*-Centered=>
# characters=>Harry P\.=>Harry Potter
-
-[test1.com:txt]
-
-## If necessary, you can define [<site>:<format>] sections to
-## customize the formats differently for the same site. Overrides
-## defaults, format and site.
-
-extratags: FanFiction,Testing,Text
-
-[test1.com:html]
-
-extratags: FanFiction,Testing,HTML
-
[teststory:defaults]
valid_entries:title,author_list,authorId_list,authorUrl_list,storyUrl,
@@ -1553,9 +1548,10 @@
extra_valid_entries:fandoms, freeformtags, freefromtags,
ao3categories, comments, chapterslashtotal, chapterstotal, kudos,
hits, bookmarks, collections, byline, bookmarked, bookmarktags,
- bookmarksummary, bookmarkprivate, bookmarkrec, restricted, series00,
- series01, series02, series03, series00Url, series01Url, series02Url,
- series03Url, series00HTML, series01HTML, series02HTML, series03HTML
+ bookmarksummary, bookmarkprivate, bookmarkrec, subscribed,
+ markedforlater, restricted, series00, series01, series02, series03,
+ series00Url, series01Url, series02Url, series03Url, series00HTML,
+ series01HTML, series02HTML, series03HTML
fandoms_label:Fandoms
freeformtags_label:Freeform Tags
freefromtags_label:Freeform Tags
@@ -1575,6 +1571,8 @@
bookmarksummary_label:My Bookmark Summary
bookmarkprivate_label:My Bookmark Private
bookmarkrec_label:My Bookmark Rec
+markedforlater_label:I Marked for Later
+subscribed_label:I Subscribed
restricted_label:Restricted to Registered Users
series00HTML_label:Series
series01HTML_label:Additional Series
@@ -2171,6 +2169,12 @@
## or just use the text. If this can't be done, the full title is used.
clean_chapter_titles: false
+## Some stories mistakenly include 'Ch' or 'Pt' at the end of the
+## story title. Appears to be a site bug or common author error. Copy
+## these to your personal.ini (and uncomment) to correct.
+#add_to_replace_metadata:
+# title=> (Ch|Pt)$=>
+
## Add the chapter description at the start of each chapter.
description_in_chapter: false
@@ -2507,9 +2511,10 @@
extra_valid_entries:fandoms, freeformtags, freefromtags,
ao3categories, comments, chapterslashtotal, chapterstotal, kudos,
hits, bookmarks, collections, byline, bookmarked, bookmarktags,
- bookmarksummary, bookmarkprivate, bookmarkrec, restricted, series00,
- series01, series02, series03, series00Url, series01Url, series02Url,
- series03Url, series00HTML, series01HTML, series02HTML, series03HTML
+ bookmarksummary, bookmarkprivate, bookmarkrec, subscribed,
+ markedforlater, restricted, series00, series01, series02, series03,
+ series00Url, series01Url, series02Url, series03Url, series00HTML,
+ series01HTML, series02HTML, series03HTML
fandoms_label:Fandoms
freeformtags_label:Freeform Tags
freefromtags_label:Freeform Tags
@@ -2529,6 +2534,8 @@
bookmarksummary_label:My Bookmark Summary
bookmarkprivate_label:My Bookmark Private
bookmarkrec_label:My Bookmark Rec
+markedforlater_label:I Marked for Later
+subscribed_label:I Subscribed
restricted_label:Restricted to Registered Users
series00HTML_label:Series
series01HTML_label:Additional Series
@@ -3540,7 +3547,10 @@
[www.storiesofarda.com]
## Some sites do not require a login, but do require the user to
## confirm they are adult for adult content. In commandline version,
-## this should go in your personal.ini, not defaults.ini.
+## this should go in your personal.ini, not defaults.ini. This is
+## more important for storiesofarda.com than most sites.
+## storiesofarda.com puts 'are you adult' question in front of
+## individual chapters, not stories.
#is_adult:true
## Site dedicated to these categories/characters/ships
@@ -3548,6 +3558,24 @@
website_encodings:Windows-1252,utf8
+[www.sunnydaleafterdark.com]
+## Site dedicated to these categories/characters/ships
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config. In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content. In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+extracategories:Buffy: The Vampire Slayer
+extracharacters:Buffy, Spike
+extraships:Spike/Buffy
+
[www.swi.org.ru]
use_basic_cache:true
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/FanFicFare-4.25.0/fanficfare/fetchers/cache_browser.py
new/FanFicFare-4.27.0/fanficfare/fetchers/cache_browser.py
--- old/FanFicFare-4.25.0/fanficfare/fetchers/cache_browser.py 2023-07-03
22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/fetchers/cache_browser.py 2023-09-01
15:30:19.000000000 +0200
@@ -70,7 +70,7 @@
# logger.debug("First time for (%s) extra
sleep"%parsedUrl.netloc)
# time.sleep(10)
fromcache=False
- read_try_sleeps = [2, 2, 2, 2, 2]
+ read_try_sleeps = [2, 2, 4, 5, 6]
while not d and read_try_sleeps:
time.sleep(read_try_sleeps.pop(0))
logger.debug("Checking for cache...")
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/FanFicFare-4.25.0/fanficfare/fetchers/fetcher_flaresolverr_proxy.py
new/FanFicFare-4.27.0/fanficfare/fetchers/fetcher_flaresolverr_proxy.py
--- old/FanFicFare-4.25.0/fanficfare/fetchers/fetcher_flaresolverr_proxy.py
2023-07-03 22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/fanficfare/fetchers/fetcher_flaresolverr_proxy.py
2023-09-01 15:30:19.000000000 +0200
@@ -74,7 +74,7 @@
'maxTimeout':
int(self.getConfig("flaresolverr_proxy_timeout","60000")),
# download:True causes response to be base64 encoded
# which makes images work.
- 'cookies':cookiejar_to_jsonable(self.get_cookiejar()),
+
'cookies':cookiejar_to_jsonable(filter_cookies(self.get_cookiejar(),url)),
'postData':encode_params(parameters),
}
if self.getConfig('use_flaresolverr_proxy') == 'withimages':
@@ -166,6 +166,17 @@
url,
False)
+## flaresolverr passes *all* cookies, not just domain appropriate
+## ones.
+def filter_cookies(cookiejar,url):
+ retval = []
+ logger.debug("url:%s"%url)
+ for c in cookiejar:
+ logger.debug("domain: %s"%c.domain)
+ if c.domain in url:
+ retval.append(c)
+ return retval
+
def cookiejar_to_jsonable(cookiejar):
retval = []
for c in cookiejar:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/FanFicFare-4.25.0/fanficfare/story.py
new/FanFicFare-4.27.0/fanficfare/story.py
--- old/FanFicFare-4.25.0/fanficfare/story.py 2023-07-03 22:38:14.000000000
+0200
+++ new/FanFicFare-4.27.0/fanficfare/story.py 2023-09-01 15:30:19.000000000
+0200
@@ -17,7 +17,7 @@
from __future__ import absolute_import
import os, re, sys
-from collections import defaultdict
+from collections import defaultdict, OrderedDict
import string
import datetime
from math import floor
@@ -1103,8 +1103,6 @@
self.getMetadata('storyUrl',
removeallentities, doreplacements),
self.getMetadata('title',
removeallentities, doreplacements)))
- self.extendList("extratags",self.getConfigList("extratags"))
-
series = self.getMetadata('series', removeallentities, doreplacements)
seriesUrl = self.getMetadata('seriesUrl', removeallentities,
doreplacements)
if series:
@@ -1276,6 +1274,11 @@
and self.getConfig('add_genre_when_multi_category') not in
retlist ):
retlist.append(self.getConfig('add_genre_when_multi_category'))
+ if listname == 'extratags' and not retlist:
+ ## extratags comes directly from .ini Here to allow
+ ## include_in_extratags to still work for completeness
+ retlist = self.getConfigList('extratags')
+
if retlist:
if doreplacements:
newretlist = []
@@ -1325,35 +1328,53 @@
return retlist
def getSubjectTags(self, removeallentities=False):
- # set to avoid duplicates subject tags.
- subjectset = set()
+ ## Used both to populate epub <dc:subject> tags and Calibre
+ ## Tags column.
- tags_list = self.getConfigList("include_subject_tags") +
self.getConfigList("extra_subject_tags")
+ # to avoid duplicates subject tags *and* allow order of
+ # <dc:subject> tags.
+ subjectset = OrderedDict()
- # metadata all go into dc:subject tags, but only if they are
configured.
- for (name,value) in
six.iteritems(self.getAllMetadata(removeallentities=removeallentities,keeplists=True)):
- if name+'.SPLIT' in tags_list:
- flist=[]
- if isinstance(value,list):
- for tag in value:
- flist.extend(tag.split(','))
- else:
- flist.extend(value)
- for tag in flist:
- subjectset.add(tag)
- elif name in tags_list:
- if isinstance(value,list):
- for tag in value:
- subjectset.add(tag)
- else:
- subjectset.add(value)
+ tags_list = self.getConfigList("include_subject_tags") +
self.getConfigList("extra_subject_tags")
- if None in subjectset:
- subjectset.remove(None)
- if '' in subjectset:
- subjectset.remove('')
+ ## This used to spin on keys of self.getAllMetadata() look for
+ ## key in tags_list. No idea why, probably from even older code
+ ## 4bb91cd0c5877cf64a779c92d1f9f338a130fa5b
+ ## Found a reason, if a poor one. extratags was populated in
+ ## story as a side-effect. Moved to getList()
+ for entry in tags_list:
+ ## allow both _LIST and .SPLIT
+ entry_key = entry.replace('_LIST','').replace('.SPLIT','')
+ if not self.isValidMetaEntry(entry_key):
+ logger.warning("Skipping invalid metadata entry (%s) in
include_subject_tags",entry)
+ continue
+ if '_LIST' in entry:
+ # _LIST indicates to use the whole list (as a string)
+ # to match _LIST in replace_metadata. Then split by
+ # comma(,)
+ value_list = [ self.getMetadata(entry_key,
+
removeallentities=removeallentities) ]
+ else:
+ # use lists to match prior behavior, skipping _LIST
+ # replace_metadata lines.
+ value_list = self.getList(entry_key,
+ removeallentities=removeallentities)
+ if '.SPLIT' in entry:
+ # split each entry value by ',' .SPLIT was basically
+ # obsoleted by \, splitting, but may still be used in
+ # some users' config. Could also be useful with _LIST
+ # now.
+ split_list = []
+ for value in value_list:
+ split_list.extend(value.split(','))
+ value_list = split_list
+
+ for v in [ x.strip() for x in value_list ]:
+ if v: # skip '' or None
+ subjectset[v] = True
- return list(subjectset)
+ logger.debug("getSubjectTags:%s"%subjectset.keys())
+ return list(subjectset.keys())
def addChapter(self, chap, newchap=False):
# logger.debug("addChapter(%s,%s)"%(chap,newchap))
@@ -1382,7 +1403,7 @@
addnums = len(self.chapters) > 1 and (
self.getConfig('add_chapter_numbers') == "true"
or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)
)
- marknew = len(self.chapters) > 1 and
self.getConfig('mark_new_chapters') # true or latestonly
+ marknew = self.getConfig('mark_new_chapters') # true or latestonly
defpattern = self.getConfig('chapter_title_def_pattern','${title}') #
default val in case of missing defaults.ini
if addnums and marknew:
@@ -1409,7 +1430,7 @@
toctempl = string.Template(tocpattern)
for index, chap in enumerate(self.chapters):
- if chap['new']:
+ if chap['new'] or self.getMetadata('newforanthology'):
usetempl = newtempl
else:
usetempl = templ
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/FanFicFare-4.25.0/setup.py
new/FanFicFare-4.27.0/setup.py
--- old/FanFicFare-4.25.0/setup.py 2023-07-03 22:38:14.000000000 +0200
+++ new/FanFicFare-4.27.0/setup.py 2023-09-01 15:30:19.000000000 +0200
@@ -26,7 +26,7 @@
name=package_name,
# Versions should comply with PEP440.
- version="4.25.0",
+ version="4.27.0",
description='A tool for downloading fanfiction to eBook formats',
long_description=long_description,
