Hello community, here is the log from the commit of package python-fanficfare for openSUSE:Leap:15.2 checked in at 2020-03-15 13:35:34 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Leap:15.2/python-fanficfare (Old) and /work/SRC/openSUSE:Leap:15.2/.python-fanficfare.new.3160 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-fanficfare" Sun Mar 15 13:35:34 2020 rev:8 rq:783170 version:3.16.0 Changes: -------- --- /work/SRC/openSUSE:Leap:15.2/python-fanficfare/python-fanficfare.changes 2020-03-09 18:05:13.716813437 +0100 +++ /work/SRC/openSUSE:Leap:15.2/.python-fanficfare.new.3160/python-fanficfare.changes 2020-03-15 13:35:35.298810440 +0100 @@ -1,0 +2,16 @@ +Mon Mar 2 18:34:01 CET 2020 - Matej Cepl <[email protected]> + +- Update 3.16.0: + - New Site: archive.hpfanfictalk.com (eFiction non-base) + - Reduce debug output in base_xenforoforum_adapter. + - Add replace_xbr_with_hr feature. + - Update translations. + - Fix for adapter_storiesonlinenet requiring 'v' from + login.php. + - Add more domains for AO3. + - Use storyUrl from metadata for checking library, for + those sites that make canonical storyUrl difficult, like + adapter_literotica. Closes #461. Plugin only. + - Optional EPUB 3.0 output + +------------------------------------------------------------------- Old: ---- FanFicFare-3.15.0.tar.gz New: ---- FanFicFare-3.16.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-fanficfare.spec ++++++ --- /var/tmp/diff_new_pack.MJxSrl/_old 2020-03-15 13:35:35.890810792 +0100 +++ /var/tmp/diff_new_pack.MJxSrl/_new 2020-03-15 13:35:35.894810795 +0100 @@ -21,7 +21,7 @@ %define skip_python2 1 %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-fanficfare -Version: 3.15.0 +Version: 3.16.0 Release: 0 Summary: Tool for making eBooks from stories on fanfiction and other web sites License: GPL-3.0-only ++++++ FanFicFare-3.15.0.tar.gz -> FanFicFare-3.16.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/calibre-plugin/__init__.py new/FanFicFare-3.16.0/calibre-plugin/__init__.py --- old/FanFicFare-3.15.0/calibre-plugin/__init__.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/calibre-plugin/__init__.py 2020-02-13 18:50:38.000000000 +0100 @@ -33,7 +33,7 @@ from calibre.customize import InterfaceActionBase # pulled out from FanFicFareBase for saving in prefs.py -__version__ = (3, 15, 0) +__version__ = (3, 16, 0) ## Apparently the name for this class doesn't matter--it was still ## 'demo' for the first few versions. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/calibre-plugin/fff_plugin.py new/FanFicFare-3.16.0/calibre-plugin/fff_plugin.py --- old/FanFicFare-3.15.0/calibre-plugin/fff_plugin.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/calibre-plugin/fff_plugin.py 2020-02-13 18:50:38.000000000 +0100 @@ -1177,7 +1177,7 @@ story = self.get_story_metadata_only(adapter) book['title'] = story.getMetadata('title') book['author'] = [story.getMetadata('author')] - book['url'] = story.getMetadata('storyUrl', removeallentities=True) + url = book['url'] = story.getMetadata('storyUrl', removeallentities=True) ## Check reject list. Redundant with below for when story ## URL changes, but also kept here to avoid network hit in @@ -1209,7 +1209,7 @@ book['comment'] = _("Story in Series Anthology(%s).")%series book['title'] = story.getMetadata('title') book['author'] = [story.getMetadata('author')] - book['url'] = story.getMetadata('storyUrl', removeallentities=True) + url = book['url'] = story.getMetadata('storyUrl', removeallentities=True) book['good']=False book['icon']='rotate-right.png' book['status'] = _('Skipped') @@ -1238,7 +1238,7 @@ book['title'] = story.getMetadata("title", removeallentities=True) book['author_sort'] = book['author'] = story.getList("author", removeallentities=True) book['publisher'] = story.getMetadata("publisher") - book['url'] = story.getMetadata("storyUrl", removeallentities=True) + url = book['url'] = story.getMetadata("storyUrl", removeallentities=True) book['tags'] = story.getSubjectTags(removeallentities=True) book['comments'] = story.get_sanitized_description() book['series'] = story.getMetadata("series", removeallentities=True) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/calibre-plugin/plugin-defaults.ini new/FanFicFare-3.16.0/calibre-plugin/plugin-defaults.ini --- old/FanFicFare-3.15.0/calibre-plugin/plugin-defaults.ini 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/calibre-plugin/plugin-defaults.ini 2020-02-13 18:50:38.000000000 +0100 @@ -268,6 +268,11 @@ ## those stories. #dedup_chapter_list:false +## Some sites/authors/stories use several br tags for scene/section +## breaks. When set replace_xbr_with_hr:X will cause FFF to search +## for X or more consecutive br tags and replace them with br br hr br. +#replace_xbr_with_hr:3 + ## Some readers don't show horizontal rule (<hr />) tags correctly. ## This replaces them all with a centered '* * *'. (Note centering ## doesn't work on some devices either.) @@ -817,6 +822,14 @@ ## Each output format has a section that overrides [defaults] +## As of Jan 2020, FanFicFare can output either epub 2.0 or 3.0 (with +## backward compatibility toc.ncx file). 2.0 will still be the +## default for now. +epub_version:2.0 + +## epub is already a zip file. +zip_output: false + ## epub carries the TOC in metadata. ## mobi generated from epub by calibre will have a TOC at the end. include_tocpage: false @@ -1135,6 +1148,65 @@ #username:YourName #password:yourpassword +[archive.hpfanfictalk.com] +## Some sites also require the user to confirm they are adult for +## adult content. In commandline version, this should go in your +## personal.ini, not defaults.ini. +#is_adult:true + +add_to_extra_valid_entries:,themes,inclusivity,house, + series00,series00Url,series00HTML, + series01,series01Url,series01HTML, + series02,series02Url,series02HTML, + series03,series03Url,series03HTML, + series04,series04Url,series04HTML, + series05,series05Url,series05HTML, + series06,series06Url,series06HTML, + series07,series07Url,series07HTML, + series08,series08Url,series08HTML, + series09,series09Url,series09HTML, + +## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to +## make entryHTML. +make_linkhtml_entries:series00,series01,series02,series03,series04, + series05,series06,series07,series08,series09 + +themes_label:Themes +inclusivity_label:Inclusivity +house_label:HPFT Forum House + +## series00 will be the same as common metadata series. +series00HTML_label:Series +series01HTML_label:Additional Series +series02HTML_label:Additional Series +series03HTML_label:Additional Series +series04HTML_label:Additional Series +series05HTML_label:Additional Series +series06HTML_label:Additional Series +series07HTML_label:Additional Series +series08HTML_label:Additional Series +series09HTML_label:Additional Series + +## Try to collect series names and numbers of this story in those +## series. This lets us turn it on and off by site without keeping a +## lengthy titlepage_entries per site and prevents it updating in the +## plugin. +collect_series: true + +add_to_extra_titlepage_entries:,series01HTML,series02HTML,series03HTML, + series04HTML,series05HTML,series06HTML,series07HTML,series08HTML,series09HTML + +## archive.hpfanfictalk.com takes margins away, even from p tags, by +## default. So authors have to either include extra br/p tags or +## their own styles. These allow for both, but leave you at the mercy +## of author CSS. +add_to_output_css: + * { + margin: 0; + padding: 0; + } +add_to_keep_html_attrs:,style + [archive.shriftweb.org] website_encodings:Windows-1252,utf8,iso-8859-1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/calibre-plugin/translations/et.po new/FanFicFare-3.16.0/calibre-plugin/translations/et.po --- old/FanFicFare-3.15.0/calibre-plugin/translations/et.po 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/calibre-plugin/translations/et.po 2020-02-13 18:50:38.000000000 +0100 @@ -2,12 +2,12 @@ # Copyright (C) YEAR ORGANIZATION # # Translators: -# Maidur, 2016-2019 +# Maidur, 2016-2020 msgid "" msgstr "" "Project-Id-Version: calibre-plugins\n" "POT-Creation-Date: 2019-11-29 09:28+Central Standard Time\n" -"PO-Revision-Date: 2019-12-28 22:12+0000\n" +"PO-Revision-Date: 2020-01-28 08:45+0000\n" "Last-Translator: Maidur\n" "Language-Team: Estonian (http://www.transifex.com/calibre/calibre-plugins/language/et/)\n" "MIME-Version: 1.0\n" @@ -210,7 +210,7 @@ "adult--stories that need those will just fail.<br />Only available for " "Update/Overwrite of existing books in case URL given isn't canonical or " "matches to existing book by Title/Author." -msgstr "" +msgstr "Igal allalaadimisel pakub FanFicFare valikut korjata saitidelt metaandmeid taustal töötavas protsessis.<br />See annab uuendades sulle kontrolli tagasi varem, kuid sult ei küsita kasutajanime/parooli ega seda, kas oled täisealine -- neid vajavad jutud lihtsalt ebaõnnestuvad.<br />Ainult saadaval olemasolevate raamatute Uuendamisel/Ülekirjutamisel, kui antud URL pole kanooniline või vastab olemasoleva raamatu Pealkirja/Autoriga." #: config.py:490 msgid "Updating Calibre Options" @@ -358,7 +358,7 @@ msgid "" "The FanFicFare toolbar button will bring up the plugin menu. If unchecked, " "it will <i>Download from URLs</i> or optionally Update, see below." -msgstr "" +msgstr "FanFicFare tööriistariba nupp avab plugina menüü. Kui märgistamata, teostab see tegevuse <i>Laadi alla URLidelt</i> või valikuliselt Uuenda, vaata allpool." #: config.py:585 msgid "Default to Update when books selected?" @@ -368,7 +368,7 @@ msgid "" "The FanFicFare toolbar button will Update if books are selected. If " "unchecked, it will always <i>Download from URLs</i>." -msgstr "" +msgstr "FanFicFare tööriistariba nupp Uuendab, kui raamatuid on valitud. Kui märgistamata, teostab see alati tegevuse <i>Laadi alla URLidelt</i>." #: config.py:595 msgid "Keep 'Add New from URL(s)' dialog on top?" @@ -713,7 +713,7 @@ "Calibre's Polish feature will be used to inject or update the generated " "cover into the EPUB ebook file.<br />Used for both Calibre and Plugin " "generated covers." -msgstr "" +msgstr "Kasutatakse Calibre Viimistlemise funktsiooni, et loodud kaanepilt EPUB e-raamatu faili sisestada või uuendada.<br />Kasutatakse nii Calibre kui Plugina loodud kaanepiltide jaoks." #: config.py:1061 msgid "%(gc)s(Plugin) Settings" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/adapters/__init__.py new/FanFicFare-3.16.0/fanficfare/adapters/__init__.py --- old/FanFicFare-3.15.0/fanficfare/adapters/__init__.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/adapters/__init__.py 2020-02-13 18:50:38.000000000 +0100 @@ -165,6 +165,7 @@ from . import adapter_mugglenetfanfictioncom from . import adapter_swiorgru from . import adapter_fanficsme +from . import adapter_archivehpfanfictalkcom ## This bit of complexity allows adapters to be added by just adding ## importing. It eliminates the long if/else clauses we used to need diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/adapters/adapter_archivehpfanfictalkcom.py new/FanFicFare-3.16.0/fanficfare/adapters/adapter_archivehpfanfictalkcom.py --- old/FanFicFare-3.15.0/fanficfare/adapters/adapter_archivehpfanfictalkcom.py 1970-01-01 01:00:00.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/adapters/adapter_archivehpfanfictalkcom.py 2020-02-13 18:50:38.000000000 +0100 @@ -0,0 +1,234 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team, 2020 FanFicFare team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +from __future__ import absolute_import +import logging +logger = logging.getLogger(__name__) +import re +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +# py2 vs py3 transition +from ..six import text_type as unicode +from ..six.moves.urllib.error import HTTPError + +from .base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return ArchiveHPfanfictalkComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class ArchiveHPfanfictalkComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','ahpfftc') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d %b %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'archive.hpfanfictalk.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + def use_pagecache(self): + ''' + adapters that will work with the page cache need to implement + this and change it to True. + ''' + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=3" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except HTTPError as e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + ## Title and author + # use BeautifulSoup HTML parser to make everything easier to find. + soup = self.make_soup(data) + # logger.debug(soup) + + # Now go hunting for all the meta data and the chapter list. + + pagetitle = soup.find('h3') + # logger.debug(pagetitle) + ## Title + a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',stripHTML(a)) + + # Find the chapters: + for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']) + + listbox = soup.find('div', {'class':'listbox'}) + # this site has two divs with class=gb-50 and no immediate container. + gb50s = soup.find_all('div', {'class':'gb-50'}) + + def list_from_urls(parent, regex, metadata): + urls = parent.find_all('a',href=re.compile(regex)) + for url in urls: + self.story.addToList(metadata,stripHTML(url)) + + list_from_urls(listbox,r'browse.php\?type=categories','category') + list_from_urls(gb50s[0],r'browse.php\?type=characters','characters') + list_from_urls(gb50s[0],r'browse.php\?type=class&type_id=11','ships') + list_from_urls(gb50s[0],r'browse.php\?type=class&type_id=14','house') + list_from_urls(gb50s[1],r'browse.php\?type=class&type_id=4','genre') + list_from_urls(gb50s[1],r'browse.php\?type=class&type_id=13','themes') + list_from_urls(gb50s[1],r'browse.php\?type=class&type_id=8','warnings') + list_from_urls(gb50s[1],r'browse.php\?type=class&type_id=10','inclusivity') + + bq = soup.find('blockquote2') + if bq: + # blockquote2??? Whatever. But we're changing it to a real tag. + bq.name='div' + self.setDescription(url,bq) + + # usually use something more precise for label search, but + # site doesn't group much. + labels = soup.find_all('b') + for labelspan in labels: + # logger.debug(labelspan) + value = labelspan.nextSibling + label = stripHTML(labelspan) + # logger.debug(value) + # logger.debug(label) + + if 'Rating' in label: + # Mature Audiences · Incomplete + (rating,status) = value.split('·') + self.story.setMetadata('rating', rating) + if 'Complete' in status: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Story Length' in label: + stripHTML(value) + # 10 chapters (45462 words) + v = stripHTML(value) + v = v.split('(')[1] + v = v.split(' words')[0] + self.story.setMetadata('numWords', v) + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value).replace('·',''), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + # Site allows stories to be in several series at once. FFF + # isn't thrilled with that, we have series00, series01, etc. + # Example: + # http://archive.hpfanfictalk.com/viewstory.php?sid=483 + + if self.getConfig("collect_series"): + seriesspan = soup.find('span',label='Series') + for i, seriesa in enumerate(seriesspan.find_all('a', href=re.compile(r"viewseries\.php\?seriesid=\d+"))): + # logger.debug(seriesa) + series_name = stripHTML(seriesa) + series_url = 'https://'+self.host+'/'+seriesa['href'] + + seriessoup = self.make_soup(self._fetchUrl(series_url)) + storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+')) + # logger.debug(storyas) + j=1 + found = False + for storya in storyas: + # logger.debug(storya) + ## allow for JS links. + if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in storya['href']: + found = True + break + j+=1 + if found: + series_index = j + self.story.setMetadata('series%02d'%i,"%s [%s]"%(series_name,series_index)) + self.story.setMetadata('series%02dUrl'%i,series_url) + if i == 0: + self.setSeries(series_name, series_index) + self.story.setMetadata('seriesUrl',series_url) + else: + logger.debug("Story URL not found in series (%s) page, not including."%series_url) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = self.make_soup(self._fetchUrl(url)) + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/adapters/adapter_archiveofourownorg.py new/FanFicFare-3.16.0/fanficfare/adapters/adapter_archiveofourownorg.py --- old/FanFicFare-3.15.0/fanficfare/adapters/adapter_archiveofourownorg.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/adapters/adapter_archiveofourownorg.py 2020-02-13 18:50:38.000000000 +0100 @@ -77,7 +77,14 @@ @classmethod def getAcceptDomains(cls): - return ['archiveofourown.org','archiveofourown.com','download.archiveofourown.org','download.archiveofourown.com'] + return ['archiveofourown.org', + 'archiveofourown.com', + 'archiveofourown.net', + 'download.archiveofourown.org', + 'download.archiveofourown.com', + 'download.archiveofourown.net', + 'ao3.org', + ] @classmethod def getSiteExampleURLs(cls): @@ -86,7 +93,15 @@ def getSiteURLPattern(self): # https://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770 # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs. - return r"https?://(download\.)?archiveofourown\.(org|com)(/collections/[^/]+)?/works/0*(?P<id>\d+)" + logger.debug(r"https?://" + r"|".join([x.replace('.','\.') for x in self.getAcceptDomains()]) + r"(/collections/[^/]+)?/works/0*(?P<id>\d+)") + return r"https?://(" + r"|".join([x.replace('.','\.') for x in self.getAcceptDomains()]) + r")(/collections/[^/]+)?/works/0*(?P<id>\d+)" + + # The certificate is only valid for the following names: + # ao3.org, + # archiveofourown.com, + # archiveofourown.net, + # archiveofourown.org, + # www.ao3.org, ## Login def needToLoginCheck(self, data): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/adapters/adapter_phoenixsongnet.py new/FanFicFare-3.16.0/fanficfare/adapters/adapter_phoenixsongnet.py --- old/FanFicFare-3.15.0/fanficfare/adapters/adapter_phoenixsongnet.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/adapters/adapter_phoenixsongnet.py 2020-02-13 18:50:38.000000000 +0100 @@ -212,7 +212,7 @@ soup = self.make_soup(self._fetchUrl(url)) chapter=self.make_soup('<div class="story"></div>') - for p in soup.findAll('p'): + for p in soup.findAll(['p','blockquote']): if "This is for problems with the formatting or the layout of the chapter." in stripHTML(p): break chapter.append(p) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/adapters/adapter_storiesonlinenet.py new/FanFicFare-3.16.0/fanficfare/adapters/adapter_storiesonlinenet.py --- old/FanFicFare-3.15.0/fanficfare/adapters/adapter_storiesonlinenet.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/adapters/adapter_storiesonlinenet.py 2020-02-13 18:50:38.000000000 +0100 @@ -113,6 +113,10 @@ logger.info("Login Required for URL %s" % loginUrl) raise exceptions.FailedToLogin(url,params['theusername']) + ## fetch 'v' code from login page. + soup = self.make_soup(self._fetchUrl(loginUrl,usecache=False)) + params['v']=soup.find('input', {'name':'v'})['value'] + try: d = self._postUrl(loginUrl,params,usecache=False) self.needToLogin = False diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/adapters/adapter_test1.py new/FanFicFare-3.16.0/fanficfare/adapters/adapter_test1.py --- old/FanFicFare-3.15.0/fanficfare/adapters/adapter_test1.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/adapters/adapter_test1.py 2020-02-13 18:50:38.000000000 +0100 @@ -414,6 +414,17 @@ horizontal rules <hr size=1 noshade> <p>"Lorem ipsum dolor sit amet", consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore--et dolore magna aliqua. 'Ut enim ad minim veniam', quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p> +<br> +<br> +Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.<br/> +<br/> +<br/> +<br> +<br> + <br/> +<br/> <br/> +<br/> +"Lorem ipsum dolor sit amet", consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore--et dolore magna aliqua. 'Ut enim ad minim veniam', quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.<br> <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p> </div> ''' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/adapters/base_adapter.py new/FanFicFare-3.16.0/fanficfare/adapters/base_adapter.py --- old/FanFicFare-3.15.0/fanficfare/adapters/base_adapter.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/adapters/base_adapter.py 2020-02-13 18:50:38.000000000 +0100 @@ -262,6 +262,7 @@ passchap['title'] = title passchap['html'] = data + ## XXX -- add chapter text replacement here? self.story.addChapter(passchap, newchap) self.storyDone = True @@ -541,6 +542,16 @@ # This is primarily for epub updates. retval = re.sub(r"</?(html|head|body)[^>]*>\r?\n?","",retval) + try: + xbr = int(self.getConfig("replace_xbr_with_hr",default=0)) + if xbr > 0: + start = datetime.now() + retval = re.sub('(\s*<br[^>]*>\s*){%d,}'%xbr, + '<br/>\n<br/>\n<hr/>\n<br/>',retval) + self.times.add("utf8FromSoup->replace_xbr_with_hr", datetime.now() - start) + except: + logger.debug("Ignoring non-int replace_xbr_with_hr(%s)"%self.getConfig("replace_xbr_with_hr")) + if self.getConfig("replace_br_with_p") and allow_replace_br_with_p: # Apply heuristic processing to replace <br> paragraph # breaks with <p> tags. @@ -553,6 +564,7 @@ # soup is more difficult than it first appears. So cheat. retval = re.sub("<hr[^>]*>","<div class='center'>* * *</div>",retval) + return retval def make_soup(self,data): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/adapters/base_xenforoforum_adapter.py new/FanFicFare-3.16.0/fanficfare/adapters/base_xenforoforum_adapter.py --- old/FanFicFare-3.15.0/fanficfare/adapters/base_xenforoforum_adapter.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/adapters/base_xenforoforum_adapter.py 2020-02-13 18:50:38.000000000 +0100 @@ -347,7 +347,7 @@ def fetch_threadmarks(self,url,tmcat_name,tmcat_num, passed_tmcat_index=0, dedup=[]): threadmarks=[] if url in dedup: - logger.debug("fetch_threadmarks(%s,tmcat_num=%s,passed_tmcat_index:%s,url=%s,dedup=%s)\nDuplicate threadmark URL, skipping"%(tmcat_name,tmcat_num, passed_tmcat_index, url, dedup)) + # logger.debug("fetch_threadmarks(%s,tmcat_num=%s,passed_tmcat_index:%s,url=%s,dedup=%s)\nDuplicate threadmark URL, skipping"%(tmcat_name,tmcat_num, passed_tmcat_index, url, dedup)) return threadmarks dedup = dedup + [url] soupmarks = self.make_soup(self._fetchUrl(url)) @@ -632,7 +632,7 @@ datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours. return makeDate(datestr, self.dateformat) except: - logger.debug('No date found in %s, going on without'%parenttag,exc_info=True) + # logger.debug('No date found in %s, going on without'%parenttag,exc_info=True) return None def cache_posts(self,topsoup): @@ -667,7 +667,7 @@ self.getConfig("use_reader_mode",True) and '/threads/' not in url and (index > 0 or not self.getConfig('always_include_first_post')) ): - logger.debug("USE READER MODE") + logger.debug("Using reader mode") # in case it changes: posts_per_page = int(self.getConfig("reader_posts_per_page",10)) @@ -678,9 +678,9 @@ if not souptag and url in self.threadmarks_for_reader: (tmcat_num,tmcat_index)=self.threadmarks_for_reader[url] reader_page_num = int((tmcat_index+posts_per_page)/posts_per_page) + offset - logger.debug('Reader page offset:%s tmcat_num:%s tmcat_index:%s'%(offset,tmcat_num,tmcat_index)) + # logger.debug('Reader page offset:%s tmcat_num:%s tmcat_index:%s'%(offset,tmcat_num,tmcat_index)) reader_url=self.make_reader_url(tmcat_num,reader_page_num) - logger.debug("Fetch reader URL to: %s"%reader_url) + # logger.debug("Fetch reader URL to: %s"%reader_url) topsoup = self.make_soup(self._fetchUrl(reader_url)) # make_soup() loads cache with posts from that reader # page. looking for it in cache reuses code in @@ -692,7 +692,7 @@ break if not souptag: - logger.debug("DON'T USE READER MODE") + logger.debug("Not using reader mode") souptag = self.get_cache_post(url) if not souptag: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/cli.py new/FanFicFare-3.16.0/fanficfare/cli.py --- old/FanFicFare-3.15.0/fanficfare/cli.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/cli.py 2020-02-13 18:50:38.000000000 +0100 @@ -40,7 +40,7 @@ def pickle_load(f): return pickle.load(f,encoding="bytes") -version="3.15.0" +version="3.16.0" os.environ['CURRENT_VERSION_ID']=version global_cache = 'global_cache' @@ -548,7 +548,7 @@ homepath = join(expanduser('~'), '.fanficdownloader') ## also look for .fanficfare now, give higher priority than old dir. homepath2 = join(expanduser('~'), '.fanficfare') - xdgpath = os.environ.get('XDG_CONFIG_HOME', default=join(expanduser('~'),'.config')) + xdgpath = os.environ.get('XDG_CONFIG_HOME', join(expanduser('~'),'.config')) xdgpath = join(xdgpath, 'fanficfare') if passed_defaultsini: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/configurable.py new/FanFicFare-3.16.0/fanficfare/configurable.py --- old/FanFicFare-3.15.0/fanficfare/configurable.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/configurable.py 2020-02-13 18:50:38.000000000 +0100 @@ -422,6 +422,7 @@ 'remove_transparency', 'replace_br_with_p', 'replace_hr', + 'replace_xbr_with_hr', 'replace_metadata', 'slow_down_sleep_time', 'sort_ships', @@ -489,6 +490,7 @@ 'ignore_chapter_url_list', 'dedup_chapter_list', 'max_zalgo', + 'epub_version', ]) # *known* entry keywords -- or rather regexps for them. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/defaults.ini new/FanFicFare-3.16.0/fanficfare/defaults.ini --- old/FanFicFare-3.15.0/fanficfare/defaults.ini 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/defaults.ini 2020-02-13 18:50:38.000000000 +0100 @@ -324,6 +324,11 @@ ## those stories. #dedup_chapter_list:false +## Some sites/authors/stories use several br tags for scene/section +## breaks. When set replace_xbr_with_hr:X will cause FFF to search +## for X or more consecutive br tags and replace them with br br hr br. +#replace_xbr_with_hr:3 + ## Some readers don't show horizontal rule (<hr />) tags correctly. ## This replaces them all with a centered '* * *'. (Note centering ## doesn't work on some devices either.) @@ -844,6 +849,11 @@ ## Each output format has a section that overrides [defaults] +## As of Jan 2020, FanFicFare can output either epub 2.0 or 3.0 (with +## backward compatibility toc.ncx file). 2.0 will still be the +## default for now. +epub_version:2.0 + ## epub is already a zip file. zip_output: false @@ -1171,6 +1181,65 @@ #username:YourName #password:yourpassword +[archive.hpfanfictalk.com] +## Some sites also require the user to confirm they are adult for +## adult content. In commandline version, this should go in your +## personal.ini, not defaults.ini. +#is_adult:true + +add_to_extra_valid_entries:,themes,inclusivity,house, + series00,series00Url,series00HTML, + series01,series01Url,series01HTML, + series02,series02Url,series02HTML, + series03,series03Url,series03HTML, + series04,series04Url,series04HTML, + series05,series05Url,series05HTML, + series06,series06Url,series06HTML, + series07,series07Url,series07HTML, + series08,series08Url,series08HTML, + series09,series09Url,series09HTML, + +## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to +## make entryHTML. +make_linkhtml_entries:series00,series01,series02,series03,series04, + series05,series06,series07,series08,series09 + +themes_label:Themes +inclusivity_label:Inclusivity +house_label:HPFT Forum House + +## series00 will be the same as common metadata series. +series00HTML_label:Series +series01HTML_label:Additional Series +series02HTML_label:Additional Series +series03HTML_label:Additional Series +series04HTML_label:Additional Series +series05HTML_label:Additional Series +series06HTML_label:Additional Series +series07HTML_label:Additional Series +series08HTML_label:Additional Series +series09HTML_label:Additional Series + +## Try to collect series names and numbers of this story in those +## series. This lets us turn it on and off by site without keeping a +## lengthy titlepage_entries per site and prevents it updating in the +## plugin. +collect_series: true + +add_to_extra_titlepage_entries:,series01HTML,series02HTML,series03HTML, + series04HTML,series05HTML,series06HTML,series07HTML,series08HTML,series09HTML + +## archive.hpfanfictalk.com takes margins away, even from p tags, by +## default. So authors have to either include extra br/p tags or +## their own styles. These allow for both, but leave you at the mercy +## of author CSS. +add_to_output_css: + * { + margin: 0; + padding: 0; + } +add_to_keep_html_attrs:,style + [archive.shriftweb.org] website_encodings:Windows-1252,utf8,iso-8859-1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/epubutils.py new/FanFicFare-3.16.0/fanficfare/epubutils.py --- old/FanFicFare-3.15.0/fanficfare/epubutils.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/epubutils.py 2020-02-13 18:50:38.000000000 +0100 @@ -2,7 +2,7 @@ from __future__ import absolute_import __license__ = 'GPL v3' -__copyright__ = '2018, Jim Miller' +__copyright__ = '2020, Jim Miller' __docformat__ = 'restructuredtext en' import logging @@ -304,10 +304,22 @@ ## logger.debug("toc.ncx zf:%s"%zf) unmerge_tocncxdoms[zf] = parseString(inputepub.read(zf)) + unmerge_navxhtmldoms = {} + ## spin through file contents, saving any unmerge toc.ncx files. + for zf in inputepub.namelist(): + ## logger.debug("zf:%s"%zf) + if zf.endswith('/nav.xhtml'): + ## logger.debug("toc.ncx zf:%s"%zf) + unmerge_navxhtmldoms[zf] = parseString(inputepub.read(zf)) + tocncxdom = parseString(inputepub.read('toc.ncx')) + if 'nav.xhtml' in inputepub.namelist(): + navxhtmldom = parseString(inputepub.read('nav.xhtml')) + else: + navxhtmldom = None ## spin through file contents. for zf in inputepub.namelist(): - if zf not in ['mimetype','toc.ncx'] and not zf.endswith('/toc.ncx'): + if zf not in ['mimetype','toc.ncx','nav.xhtml'] and not zf.endswith('/toc.ncx') and not zf.endswith('/nav.xhtml'): entrychanged = False data = inputepub.read(zf) # if isinstance(data,unicode): @@ -359,6 +371,8 @@ if entrychanged: logger.debug("\nentrychanged:%s\n"%zf) _replace_tocncx(tocncxdom,zf,chaptertoctitle) + if navxhtmldom: + _replace_navxhtml(navxhtmldom,zf,chaptertoctitle) ## Also look for and update individual ## book toc.ncx files for anthology in case ## it's unmerged. @@ -367,6 +381,8 @@ if zf_toc in unmerge_tocncxdoms: _replace_tocncx(unmerge_tocncxdoms[zf_toc],zf[mergedprefix_len:],chaptertoctitle) + if zf_toc in unmerge_navxhtmldoms: + _replace_navxhtml(unmerge_navxhtmldoms[zf_toc],zf[mergedprefix_len:],chaptertoctitle) outputepub.writestr(zf,data.encode('utf-8')) else: @@ -375,8 +391,12 @@ for tocnm, tocdom in unmerge_tocncxdoms.items(): outputepub.writestr(tocnm,tocdom.toxml(encoding='utf-8')) + for navnm, navdom in unmerge_navxhtmldoms.items(): + outputepub.writestr(navnm,navdom.toxml(encoding='utf-8')) outputepub.writestr('toc.ncx',tocncxdom.toxml(encoding='utf-8')) + if navxhtmldom: + outputepub.writestr('nav.xhtml',navxhtmldom.toxml(encoding='utf-8')) outputepub.close() # declares all the files created by Windows. otherwise, when # it runs in appengine, windows unzips the files as 000 perms. @@ -411,6 +431,15 @@ #logger.debug("text label:%s"%texttag.toxml()) continue +def _replace_navxhtml(navxhtmldom,zf,chaptertoctitle): + ## go after the TOC entry, too. + # <li><a href="OEBPS/file0003.xhtml">3. (new) Chapter 2, Sinmay on Kintikin</a></li> + for atag in navxhtmldom.getElementsByTagName("a"): + if atag.getAttribute('href') == zf: + atag.childNodes[0].replaceWholeText(chaptertoctitle) + # logger.debug("a href=%s label:%s"%(zf,atag.toxml())) + continue + def make_soup(data): ''' Convenience method for getting a bs4 soup. bs3 has been removed. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/fanficfare/writers/writer_epub.py new/FanFicFare-3.16.0/fanficfare/writers/writer_epub.py --- old/FanFicFare-3.15.0/fanficfare/writers/writer_epub.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/fanficfare/writers/writer_epub.py 2020-02-13 18:50:38.000000000 +0100 @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2011 Fanficdownloader team, 2019 FanFicFare team +# Copyright 2011 Fanficdownloader team, 2020 FanFicFare team # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -60,7 +60,7 @@ <html xmlns="http://www.w3.org/1999/xhtml";> <head> <title>${title} by ${author}</title> -<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/> +<link href="stylesheet.css" type="text/css" rel="stylesheet"/> </head> <body class="fff_titlepage"> <h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3> @@ -86,7 +86,7 @@ <html xmlns="http://www.w3.org/1999/xhtml";> <head> <title>${title} by ${author}</title> -<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/> +<link href="stylesheet.css" type="text/css" rel="stylesheet"/> </head> <body class="fff_titlepage"> <h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3> @@ -116,7 +116,7 @@ <html xmlns="http://www.w3.org/1999/xhtml";> <head> <title>${title} by ${author}</title> -<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/> +<link href="stylesheet.css" type="text/css" rel="stylesheet"/> </head> <body class="fff_tocpage"> <div> @@ -137,11 +137,11 @@ <html xmlns="http://www.w3.org/1999/xhtml";> <head> <title>${chapter}</title> -<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/> -<meta name="chapterurl" content="${url}"></meta> -<meta name="chapterorigtitle" content="${origchapter}"></meta> -<meta name="chaptertoctitle" content="${tocchapter}"></meta> -<meta name="chaptertitle" content="${chapter}"></meta> +<link href="stylesheet.css" type="text/css" rel="stylesheet"/> +<meta name="chapterurl" content="${url}" /> +<meta name="chapterorigtitle" content="${origchapter}" /> +<meta name="chaptertoctitle" content="${tocchapter}" /> +<meta name="chaptertitle" content="${chapter}" /> </head> <body class="fff_chapter"> <h3 class="fff_chapter_title">${chapter}</h3> @@ -156,7 +156,7 @@ <html xmlns="http://www.w3.org/1999/xhtml";> <head> <title>Update Log</title> -<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/> +<link href="stylesheet.css" type="text/css" rel="stylesheet"/> </head> <body class="fff_logpage"> <h3>Update Log</h3> @@ -171,7 +171,8 @@ ''') self.EPUB_LOG_UPDATE_END = string.Template(''' -</p><hr /> +</p> +<hr/> ''') self.EPUB_LOG_PAGE_END = string.Template(''' @@ -352,7 +353,13 @@ contentdom = getDOMImplementation().createDocument(None, "package", None) package = contentdom.documentElement - package.setAttribute("version","2.0") + ## might want 3.1 or something in future. + epub3 = self.getConfig("epub_version",default="2.0").startswith("3") + if epub3: + package.setAttribute("version","3.0") + else: + package.setAttribute("version","2.0") + logger.info("Saving EPUB Version "+package.getAttribute("version")) package.setAttribute("xmlns","http://www.idpf.org/2007/opf";) package.setAttribute("unique-identifier","fanficfare-uid") metadata=newTag(contentdom,"metadata", @@ -365,45 +372,65 @@ attrs={"id":"fanficfare-uid"})) if self.getMetadata('title'): - metadata.appendChild(newTag(contentdom,"dc:title",text=self.getMetadata('title'))) + metadata.appendChild(newTag(contentdom,"dc:title",text=self.getMetadata('title'), + attrs={"id":"id"})) + def creator_attrs(idnum): + if epub3: + return {"id":"id-%d"%idnum} + else: + return {"opf:role":"aut"} + idnum = 1 if self.getMetadata('author'): if self.story.isList('author'): for auth in self.story.getList('author'): metadata.appendChild(newTag(contentdom,"dc:creator", - attrs={"opf:role":"aut"}, + attrs=creator_attrs(idnum), text=auth)) + idnum += 1 else: metadata.appendChild(newTag(contentdom,"dc:creator", - attrs={"opf:role":"aut"}, + attrs=creator_attrs(idnum), text=self.getMetadata('author'))) + idnum += 1 - metadata.appendChild(newTag(contentdom,"dc:contributor",text="FanFicFare [https://github.com/JimmXinu/FanFicFare]",attrs={"opf:role":"bkp"})) - metadata.appendChild(newTag(contentdom,"dc:rights",text="")) + metadata.appendChild(newTag(contentdom,"dc:contributor",text="FanFicFare [https://github.com/JimmXinu/FanFicFare]";, + attrs={"id":"id-%d"%idnum})) + idnum += 1 + # metadata.appendChild(newTag(contentdom,"dc:rights",text="")) if self.story.getMetadata('langcode'): - metadata.appendChild(newTag(contentdom,"dc:language",text=self.story.getMetadata('langcode'))) + langcode=self.story.getMetadata('langcode') else: - metadata.appendChild(newTag(contentdom,"dc:language",text='en')) + langcode='en' + metadata.appendChild(newTag(contentdom,"dc:language",text=langcode)) # published, created, updated, calibre # Leave calling self.story.getMetadataRaw directly in case date format changes. - if self.story.getMetadataRaw('datePublished'): - metadata.appendChild(newTag(contentdom,"dc:date", - attrs={"opf:event":"publication"}, - text=self.story.getMetadataRaw('datePublished').strftime("%Y-%m-%d"))) - - if self.story.getMetadataRaw('dateCreated'): - metadata.appendChild(newTag(contentdom,"dc:date", - attrs={"opf:event":"creation"}, - text=self.story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d"))) - - if self.story.getMetadataRaw('dateUpdated'): - metadata.appendChild(newTag(contentdom,"dc:date", - attrs={"opf:event":"modification"}, - text=self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%d"))) + if epub3: + ## epub3 requires an updated modified date on every change of + ## any kind, not just *content* change. + from datetime import datetime metadata.appendChild(newTag(contentdom,"meta", - attrs={"name":"calibre:timestamp", - "content":self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%dT%H:%M:%S")})) + attrs={"property":"dcterms:modified"}, + text=datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))) + else: + if self.story.getMetadataRaw('datePublished'): + metadata.appendChild(newTag(contentdom,"dc:date", + attrs={"opf:event":"publication"}, + text=self.story.getMetadataRaw('datePublished').strftime("%Y-%m-%d"))) + + if self.story.getMetadataRaw('dateCreated'): + metadata.appendChild(newTag(contentdom,"dc:date", + attrs={"opf:event":"creation"}, + text=self.story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d"))) + + if self.story.getMetadataRaw('dateUpdated'): + metadata.appendChild(newTag(contentdom,"dc:date", + attrs={"opf:event":"modification"}, + text=self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%d"))) + metadata.appendChild(newTag(contentdom,"meta", + attrs={"name":"calibre:timestamp", + "content":self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%dT%H:%M:%S")})) series = self.story.getMetadataRaw('series') if series and self.getConfig('calibre_series_meta'): @@ -440,12 +467,43 @@ text=self.getMetadata('site'))) if self.getMetadata('storyUrl'): - metadata.appendChild(newTag(contentdom,"dc:identifier", - attrs={"opf:scheme":"URL"}, - text=self.getMetadata('storyUrl'))) + if epub3: + metadata.appendChild(newTag(contentdom,"dc:identifier", + text="URL:"+self.getMetadata('storyUrl'))) + else: + metadata.appendChild(newTag(contentdom,"dc:identifier", + attrs={"opf:scheme":"URL"}, + text=self.getMetadata('storyUrl'))) metadata.appendChild(newTag(contentdom,"dc:source", text=self.getMetadata('storyUrl'))) + if epub3: + # <meta refines="#id" property="title-type">main</meta> + metadata.appendChild(newTag(contentdom,"meta", + attrs={"property":"title-type", + "refines":"#id", + }, + text="main")) + + # epub3 removes attrs that identify dc:creator and + # dc:contributor types and instead put them here. + # 'aut' for 1-(idnum-1) + for j in range(1,idnum-1): + #<meta property="role" refines="#id-1" scheme="marc:relators">aut</meta> + metadata.appendChild(newTag(contentdom,"meta", + attrs={"property":"role", + "refines":"#id-%d"%j, + "scheme":"marc:relators", + }, + text="aut")) + + metadata.appendChild(newTag(contentdom,"meta", + attrs={"property":"role", + "refines":"#id-%d"%(idnum-1), + "scheme":"marc:relators", + }, + text="bkp")) + ## end of metadata, create manifest. items = [] # list of (id, href, type, title) tuples(all strings) itemrefs = [] # list of strings -- idrefs from .opfs' spines @@ -568,6 +626,16 @@ attrs={'id':id, 'href':href, 'media-type':type})) + if epub3: + # epub3 nav + # <item href="nav.xhtml" id="nav" media-type="application/xhtml+xml" properties="nav"/> + manifest.appendChild(newTag(contentdom,"item", + attrs={'href':'nav.xhtml', + 'id':'nav', + 'media-type':'application/xhtml+xml', + 'properties':'nav' + })) + spine = newTag(contentdom,"spine",attrs={"toc":"ncx"}) package.appendChild(spine) @@ -640,6 +708,69 @@ tocncxdom.unlink() del tocncxdom + if epub3: + ############################################################################################################## + ## create nav.xhtml file + tocnavdom = getDOMImplementation().createDocument(None, "html", None) + navxhtml = tocnavdom.documentElement + navxhtml.setAttribute("xmlns","http://www.w3.org/1999/xhtml";) + navxhtml.setAttribute("xmlns:epub","http://www.idpf.org/2007/ops";) + navxhtml.setAttribute("lang",langcode) + navxhtml.setAttribute("xml:lang",langcode) + head = tocnavdom.createElement("head") + navxhtml.appendChild(head) + head.appendChild(newTag(tocnavdom,"title",text="Navigation")) + # <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/> + head.appendChild(newTag(tocnavdom,"meta", + attrs={"http-equiv":"Content-Type", + "content":"text/html; charset=utf-8"})) + + body = tocnavdom.createElement("body") + navxhtml.appendChild(body) + nav = newTag(tocnavdom,"nav", + attrs={"epub:type":"toc"}) + body.appendChild(nav) + ol = newTag(tocnavdom,"ol") + nav.appendChild(ol) + + for item in items: + (id,href,type,title)=item + # only items to be skipped, cover.xhtml, images, toc.nav, + # stylesheet.css, should have no title. + if title: + li = newTag(tocnavdom,"li") + ol.appendChild(li) + atag = newTag(tocnavdom,"a", + attrs={"href":href}, + text=stripHTML(title)) + li.appendChild(atag) + + if self.story.cover: + # <nav epub:type="landmarks" hidden=""> + # <ol> + # <li><a href="OEBPS/cover.xhtml" epub:type="cover">Cover</a></li> + # </ol> + # </nav> + nav = newTag(tocnavdom,"nav", + attrs={"epub:type":"landmarks", + "hidden":""}) + body.appendChild(nav) + ol = newTag(tocnavdom,"ol") + nav.appendChild(ol) + li = newTag(tocnavdom,"li") + ol.appendChild(li) + atag = newTag(tocnavdom,"a", + attrs={"href":"OEBPS/cover.xhtml", + "epub:type":"cover"}, + text="Cover") + li.appendChild(atag) + + # write nav.xhtml to zip file + outputepub.writestr("nav.xhtml",tocnavdom.toxml(encoding='utf-8')) + tocnavdom.unlink() + del tocnavdom + ############################################################################################################## + # write stylesheet.css file. outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.getAllMetadata())) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/ini-order.py new/FanFicFare-3.16.0/ini-order.py --- old/FanFicFare-3.15.0/ini-order.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/ini-order.py 2020-02-13 18:50:38.000000000 +0100 @@ -61,7 +61,8 @@ kl = list(sections.keys()) kl.sort() for k in leadsects: - outfile.write("".join(sections[k])) + if k in sections: + outfile.write("".join(sections[k])) for k in kl: if k not in (leadsects + followsects): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/FanFicFare-3.15.0/setup.py new/FanFicFare-3.16.0/setup.py --- old/FanFicFare-3.15.0/setup.py 2020-01-15 16:24:33.000000000 +0100 +++ new/FanFicFare-3.16.0/setup.py 2020-02-13 18:50:38.000000000 +0100 @@ -27,7 +27,7 @@ name=package_name, # Versions should comply with PEP440. - version="3.15.0", + version="3.16.0", description='A tool for downloading fanfiction to eBook formats', long_description=long_description,
