commit 67eca412ff90708f7d02c273ec65a791cf75d23f Author: Günter Milde <mi...@lyx.org> Date: Tue Jan 23 14:01:30 2018 +0100
Fix some lyx2lyx round-trip tests. Fix cases where lyx2lyx adds changes with every round cycle uncovered by recent ctest change. Add optional `delete` argument to parser_tools.get*value(): If True, delete the matching line. More efficient "allowbreak" con/reversion. --- lib/lyx2lyx/lyx_2_0.py | 49 +++++++++++++------ lib/lyx2lyx/lyx_2_1.py | 14 +++-- lib/lyx2lyx/lyx_2_3.py | 107 +++++++++++++++++++++---------------------- lib/lyx2lyx/parser_tools.py | 25 ++++++---- 4 files changed, 109 insertions(+), 86 deletions(-) diff --git a/lib/lyx2lyx/lyx_2_0.py b/lib/lyx2lyx/lyx_2_0.py index 1741d4a..eefe0b2 100644 --- a/lib/lyx2lyx/lyx_2_0.py +++ b/lib/lyx2lyx/lyx_2_0.py @@ -22,7 +22,8 @@ import re, string import unicodedata import sys, os -from parser_tools import find_token, find_end_of, find_tokens, \ +from parser_tools import del_complete_lines, \ + find_token, find_end_of, find_tokens, \ find_token_exact, find_end_of_inset, find_end_of_layout, \ find_token_backwards, is_in_inset, get_value, get_quoted_value, \ del_token, check_token, get_option_value @@ -484,6 +485,15 @@ def revert_printindexall(document): document.body[i:k + 1] = subst i = i + 1 +strikeout_preamble = ['% for proper underlining', + r'\PassOptionsToPackage{normalem}{ulem}', + r'\usepackage{ulem}'] + +def convert_strikeout(document): + " Remove preamble code loading 'ulem' package. " + del_complete_lines(document.preamble, + ['% Added by lyx2lyx']+strikeout_preamble) + def revert_strikeout(document): " Reverts \\strikeout font attribute " @@ -491,25 +501,32 @@ def revert_strikeout(document): changed = revert_font_attrs(document.body, "\\uwave", "\\uwave") or changed changed = revert_font_attrs(document.body, "\\strikeout", "\\sout") or changed if changed == True: - insert_to_preamble(document, \ - ['% for proper underlining', - '\\PassOptionsToPackage{normalem}{ulem}', - '\\usepackage{ulem}']) + insert_to_preamble(document, strikeout_preamble) + +ulinelatex_preamble = ['% fix underbar in citations', + r'\let\cite@rig\cite', + r'\newcommand{\b@xcite}[2][\%]{\def\def@pt{\%}\def\pas@pt{#1}', + r' \mbox{\ifx\def@pt\pas@pt\cite@rig{#2}\else\cite@rig[#1]{#2}\fi}}', + r'\renewcommand{\underbar}[1]{{\let\cite\b@xcite\uline{#1}}}'] + +def convert_ulinelatex(document): + " Remove preamble code for \\uline font attribute. " + del_complete_lines(document.preamble, + ['% Added by lyx2lyx']+ulinelatex_preamble) + for line in document.preamble: + print line def revert_ulinelatex(document): - " Reverts \\uline font attribute " + " Add preamble code for \\uline font attribute in citations. " i = find_token(document.body, '\\bar under', 0) if i == -1: return - insert_to_preamble(document,\ - ['% for proper underlining', - '\\PassOptionsToPackage{normalem}{ulem}', - '\\usepackage{ulem}', - '\\let\\cite@rig\\cite', - '\\newcommand{\\b@xcite}[2][\\%]{\\def\\def@pt{\\%}\\def\\pas@pt{#1}', - ' \\mbox{\\ifx\\def@pt\\pas@pt\\cite@rig{#2}\\else\\cite@rig[#1]{#2}\\fi}}', - '\\renewcommand{\\underbar}[1]{{\\let\\cite\\b@xcite\\uline{#1}}}']) + try: + document.preamble.index(r'\usepackage{ulem}') + except ValueError: + insert_to_preamble(document, strikeout_preamble) + insert_to_preamble(document, ulinelatex_preamble) def revert_custom_processors(document): @@ -2468,9 +2485,9 @@ convert = [[346, []], [352, [convert_splitindex]], [353, []], [354, []], - [355, []], + [355, [convert_strikeout]], [356, []], - [357, []], + [357, [convert_ulinelatex]], [358, []], [359, [convert_nomencl_width]], [360, []], diff --git a/lib/lyx2lyx/lyx_2_1.py b/lib/lyx2lyx/lyx_2_1.py index f56d3ad..84b057a 100644 --- a/lib/lyx2lyx/lyx_2_1.py +++ b/lib/lyx2lyx/lyx_2_1.py @@ -24,7 +24,8 @@ import sys, os # Uncomment only what you need to import, please. -from parser_tools import count_pars_in_inset, del_token, find_token, find_token_exact, \ +from parser_tools import count_pars_in_inset, del_complete_lines, del_token, \ + find_token, find_token_exact, \ find_token_backwards, find_end_of, find_end_of_inset, find_end_of_layout, \ find_end_of_sequence, find_re, get_option_value, get_containing_layout, \ get_containing_inset, get_value, get_quoted_value, set_option_value @@ -618,15 +619,16 @@ def convert_use_package(document, pkg, commands, oldauto): # oldauto defines how the version we are converting from behaves: # if it is true, the old version uses the package automatically. # if it is false, the old version never uses the package. - i = find_token(document.header, "\\use_package", 0) + i = find_token(document.header, "\\use_package") if i == -1: document.warning("Malformed LyX document: Can't find \\use_package.") return; - j = find_token(document.preamble, "\\usepackage{" + pkg + "}", 0) - if j != -1: - # package was loaded in the preamble, convert this to header setting for round trip + packageline = "\\usepackage{%s}" % pkg + if (del_complete_lines(document.preamble, + ['% Added by lyx2lyx', packageline]) or + del_complete_lines(document.preamble, [packageline])): + # package was loaded in the preamble, convert this to header setting document.header.insert(i + 1, "\\use_package " + pkg + " 2") # on - del document.preamble[j] # If oldauto is true we have two options: # We can either set the package to auto - this is correct for files in # format 425 to 463, and may create a conflict for older files which use diff --git a/lib/lyx2lyx/lyx_2_3.py b/lib/lyx2lyx/lyx_2_3.py index 625db08..230909d 100644 --- a/lib/lyx2lyx/lyx_2_3.py +++ b/lib/lyx2lyx/lyx_2_3.py @@ -24,11 +24,11 @@ import sys, os # Uncomment only what you need to import, please. -from parser_tools import del_token, del_value, del_complete_lines, \ - find_end_of, find_end_of_layout, find_end_of_inset, find_re, \ - find_token, find_token_backwards, get_containing_layout, \ - get_bool_value, get_value, get_quoted_value -# find_tokens, find_token_exact, is_in_inset, \ +from parser_tools import (del_token, del_value, del_complete_lines, + find_complete_lines, find_end_of, find_end_of_layout, find_end_of_inset, + find_re, find_token, find_token_backwards, + get_containing_layout, get_bool_value, get_value, get_quoted_value) +# find_tokens, find_token_exact, is_in_inset, # check_token, get_option_value from lyx2lyx_tools import add_to_preamble, put_cmd_in_ert, revert_font_attrs, \ @@ -1902,6 +1902,7 @@ def convert_dashligatures(document): document.header.insert(i, "\\use_dash_ligatures %s" % str(use_dash_ligatures).lower()) + def revert_dashligatures(document): """Remove font ligature settings for en- and em-dashes. Revert conversion of \twodashes or \threedashes to literal dashes.""" @@ -1973,51 +1974,41 @@ def revert_xout(document): def convert_mathindent(document): - " add the \\is_math_indent tag " + """Add the \\is_math_indent tag. + """ + k = find_token(document.header, "\\quotes_style") # where to insert # check if the document uses the class option "fleqn" - k = find_token(document.header, "\\quotes_style", 0) - regexp = re.compile(r'^.*fleqn.*') - i = find_re(document.header, regexp, 0) - if i != -1: + options = get_value(document.header, "\\options") + if 'fleqn' in options: document.header.insert(k, "\\is_math_indent 1") - # delete the found option - document.header[i] = document.header[i].replace(",fleqn", "") - document.header[i] = document.header[i].replace(", fleqn", "") - document.header[i] = document.header[i].replace("fleqn,", "") - j = find_re(document.header, regexp, 0) - if i == j: - # then we have fleqn as the only option + # delete the fleqn option + i = find_token(document.header, "\\options") + options = [option for option in options.split(",") + if option.strip() != "fleqn"] + if options: + document.header[i] = "\\options " + ",".join(options) + else: del document.header[i] else: document.header.insert(k, "\\is_math_indent 0") - def revert_mathindent(document): " Define mathindent if set in the document " - # first output the length - regexp = re.compile(r'(\\math_indentation)') - i = find_re(document.header, regexp, 0) + # emulate and delete \math_indentation + value = get_value(document.header, "\\math_indentation", + default="default", delete=True) + if value != "default": + add_to_preamble(document, [r"\setlength{\mathindent}{%s}"%value]) + # delete \is_math_indent and emulate via document class option + if not get_bool_value(document.header, "\\is_math_indent", delete=True): + return + i = find_token(document.header, "\\options") if i != -1: - value = get_value(document.header, "\\math_indentation" , i).split()[0] - if value != "default": - add_to_preamble(document, ["\\setlength{\\mathindent}{" + value + '}']) - del document.header[i] - # now set the document class option - regexp = re.compile(r'(\\is_math_indent 1)') - i = find_re(document.header, regexp, 0) - if i == -1: - regexp = re.compile(r'(\\is_math_indent)') - j = find_re(document.header, regexp, 0) - del document.header[j] + document.header[i] = document.header[i].replace("\\options ", + "\\options fleqn,") else: - k = find_token(document.header, "\\options", 0) - if k != -1: - document.header[k] = document.header[k].replace("\\options", "\\options fleqn,") - del document.header[i] - else: - l = find_token(document.header, "\\use_default_options", 0) - document.header.insert(l, "\\options fleqn") - del document.header[i + 1] + l = find_token(document.header, "\\use_default_options") + document.header.insert(l, "\\options fleqn") def revert_baselineskip(document): @@ -2126,24 +2117,31 @@ def revert_rotfloat(document): i = i + 1 +allowbreak_emulation = [r"\begin_inset space \hspace{}", + r"\length 0dd", + r"\end_inset", + r""] + def convert_allowbreak(document): " Zero widths Space-inset -> \SpecialChar allowbreak. " - body = "\n".join(document.body) - body = body.replace("\\begin_inset space \hspace{}\n" - "\\length 0dd\n" - "\\end_inset\n\n", - "\\SpecialChar allowbreak\n") - document.body = body.split("\n") + lines = document.body + i = find_complete_lines(lines, allowbreak_emulation, 2) + while i != -1: + lines[i-1:i+4] = [lines[i-1] + r"\SpecialChar allowbreak"] + i = find_complete_lines(lines, allowbreak_emulation, i) def revert_allowbreak(document): " \SpecialChar allowbreak -> Zero widths Space-inset. " - body = "\n".join(document.body) - body = body.replace("\\SpecialChar allowbreak\n", - "\n\\begin_inset space \hspace{}\n" - "\\length 0dd\n" - "\\end_inset\n\n") - document.body = body.split("\n") + i = 1 + lines = document.body + while i < len(lines): + if lines[i].endswith(r"\SpecialChar allowbreak"): + lines[i:i+1] = [lines[i].replace(r"\SpecialChar allowbreak", "") + ] + allowbreak_emulation + i += 5 + else: + i += 1 def convert_mathnumberpos(document): @@ -2227,7 +2225,7 @@ def revert_mathnumberingname(document): document.header[i] = "\\math_number_before 0" k = find_token(document.header, "\\options", 0) if k != -1: - document.header[k] = document.header[k].replace("\\options", "\\options reqno,") + document.header[k] = document.header[k].replace("\\options", "\\options reqno,") else: l = find_token(document.header, "\\use_default_options", 0) document.header.insert(l, "\\options reqno") @@ -2240,7 +2238,8 @@ def revert_mathnumberingname(document): def convert_minted(document): " add the \\use_minted tag " - document.header.insert(-1, "\\use_minted 0") + i = find_token(document.header, "\\index ") + document.header.insert(i, "\\use_minted 0") def revert_minted(document): diff --git a/lib/lyx2lyx/parser_tools.py b/lib/lyx2lyx/parser_tools.py index 7818ac5..44914a9 100644 --- a/lib/lyx2lyx/parser_tools.py +++ b/lib/lyx2lyx/parser_tools.py @@ -53,7 +53,7 @@ find_re(lines, rexp, start[, end]): As find_token, but rexp is a regular expression object, so it has to be passed as e.g.: re.compile(r'...'). -get_value(lines, token, start[, end[, default]): +get_value(lines, token[, start[, end[, default[, delete]]]]): Similar to find_token, but it returns what follows the token on the found line. Example: get_value(document.header, "\\use_xetex", 0) @@ -64,7 +64,7 @@ get_value(lines, token, start[, end[, default]): and is what is returned if we do not find anything. So you can use that to set a default. -get_quoted_value(lines, token, start[, end[, default]]): +get_quoted_value(lines, token[, start[, end[, default[, delete]]]]): Similar to get_value, but it will strip quotes off the value, if they are present. So use this one for cases where the value is normally quoted. @@ -74,7 +74,7 @@ get_option_value(line, option): option="value" and returns value. Returns "" if not found. -get_bool_value(lines, token, start[, end[, default]]): +get_bool_value(lines, token[, start[, end[, default, delete]]]]): Like get_value, but returns a boolean. del_token(lines, token, start[, end]): @@ -357,12 +357,15 @@ def find_across_lines(lines, sub, start=0, end=0): return -1 -def get_value(lines, token, start=0, end=0, default=""): - """ get_value(lines, token, start[[, end], default]) -> string +def get_value(lines, token, start=0, end=0, default="", delete=False): + """Find `token` in `lines` and return part of line that follows it. Find the next line that looks like: token followed by other stuff - Returns "followed by other stuff" with leading and trailing + + If `delete` is True, delete the line (if found). + + Return "followed by other stuff" with leading and trailing whitespace removed. """ i = find_token_exact(lines, token, start, end) @@ -372,12 +375,14 @@ def get_value(lines, token, start=0, end=0, default=""): # return lines.pop(i)[len(token):].strip() # or default # see test_parser_tools.py l = lines[i].split(None, 1) + if delete: + del(lines[i]) if len(l) > 1: return l[1].strip() return default -def get_quoted_value(lines, token, start=0, end=0, default=""): +def get_quoted_value(lines, token, start=0, end=0, default="", delete=False): """ get_quoted_value(lines, token, start[[, end], default]) -> string Find the next line that looks like: @@ -388,13 +393,13 @@ def get_quoted_value(lines, token, start=0, end=0, default=""): if they are there. Note that we will NOT strip quotes from default! """ - val = get_value(lines, token, start, end, "") + val = get_value(lines, token, start, end, "", delete) if not val: return default return val.strip('"') -def get_bool_value(lines, token, start=0, end=0, default=None): +def get_bool_value(lines, token, start=0, end=0, default=None, delete=False): """ get_bool_value(lines, token, start[[, end], default]) -> string Find the next line that looks like: @@ -404,7 +409,7 @@ def get_bool_value(lines, token, start=0, end=0, default=None): False if bool_value is 0 or false """ - val = get_quoted_value(lines, token, start, end, "") + val = get_quoted_value(lines, token, start, end, default, delete) if val == "1" or val == "true": return True