The branch, breakrows, has been updated.
  discards  40891d7e77fe7593492b7216d0c0172c40d22253 (commit)
  discards  c04217b1dd33da1270d33f05873e057fc6b4294d (commit)
  discards  c1c34307eea3824a748ce0e4e8ceaf50ab33eb4e (commit)
  discards  2853df45e3368e92217b8aa841781583b6d07f1a (commit)
  discards  9809669eaa8dd5e6ffeb0a8acee9aa9a006e0e60 (commit)
  discards  e9d8bff4a0c32f3bde5a7b9ca0ed40156004e30d (commit)
  discards  808438295883568f0be65ef6f8e4f90f77307a16 (commit)
  discards  18b8ee331b5e6245d4fd7d1dccc2fa4d093ab67a (commit)
  discards  ff2dc8ee3c198021c7ce245452f0ce64e0e3beba (commit)
  discards  9d1e175c5eadf3852d28d2e97af79db875fccc3b (commit)
  discards  a9cb371632e1acdd0ea48d171d1626a5cc1658ea (commit)
  discards  ede249929d080f6d8745a0b988dc44afcb0f18d0 (commit)
  discards  ad6cff678912e4e17b58004ca907a845c85f5105 (commit)
  discards  951253f3968c554adefdd7a607b919778783280d (commit)
  discards  6d0f85d547e41cb48fbd08d1a56447d607cbdbe1 (commit)
  discards  25149cc84395e4ecec0e96211d7c001a45d6cfbd (commit)
  discards  31e3de583a7562c781e88a9b14f1a2af2cc5ad39 (commit)
  discards  a561d7b4538553d171a5567453c5f3aa7a1f1a37 (commit)
  discards  f314a5051713bb855d4d985df7325730790ba79d (commit)
  discards  ed6578d9ac8e49e447a1dd5296333ba47c339eb4 (commit)
  discards  2b28f7921acd8f3b2402f1dd3cb5c4a877a5eb5c (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (40891d7e77fe7593492b7216d0c0172c40d22253)
            \
             N -- N -- N (fce1597e3e8e5075fc15e083f379c0470573df71)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

- Log -----------------------------------------------------------------

commit fce1597e3e8e5075fc15e083f379c0470573df71
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Tue Oct 5 15:52:31 2021 +0200

    Increase metrics cache maximal size
    
    Increase the maximal size of the breakString cache (to compute where
    to break lines) from 512kB to 10MB. This has a big impact of cache
    hits on large file like the example in #12297, which is now 99%. On
    this example the time taken by breakString decreases from 33.5us to
    2.4us.
    
    The string width cache has been increased fro 512kB to 1MB, but this
    does not make such a big difference.
    
    Additionally, comments and variable names have been improved.
    
    Related to bug #12297.

diff --git a/src/frontends/qt/GuiFontMetrics.cpp 
b/src/frontends/qt/GuiFontMetrics.cpp
index 3feb33e..e51d40f 100644
--- a/src/frontends/qt/GuiFontMetrics.cpp
+++ b/src/frontends/qt/GuiFontMetrics.cpp
@@ -86,25 +86,27 @@ namespace lyx {
 namespace frontend {
 
 
-/*
- * Limit (strwidth|breakstr)_cache_ size to 512kB of string data.
- * Limit qtextlayout_cache_ size to 500 elements (we do not know the
- * size of the QTextLayout objects anyway).
- * Note that all these numbers are arbitrary.
- * Also, setting size to 0 is tantamount to disabling the cache.
- */
-int cache_metrics_width_size = 1 << 19;
-int cache_metrics_breakstr_size = 1 << 19;
+namespace {
+// Maximal size/cost for various caches. See QCache documentation to
+// see what cost means.
+
+// Limit strwidth_cache_ total cost to 1MB of string data.
+int const strwidth_cache_max_cost = 1024 * 1024;
+// Limit breakat_cache_ total cost to 10MB of string data.
+// This is useful for documents with very large insets.
+int const breakstr_cache_max_cost = 10 * 1024 * 1024;
 // Qt 5.x already has its own caching of QTextLayout objects
 // but it does not seem to work well on MacOS X.
 #if (QT_VERSION < 0x050000) || defined(Q_OS_MAC)
-int cache_metrics_qtextlayout_size = 500;
+// Limit qtextlayout_cache_ size to 500 elements (we do not know the
+// size of the QTextLayout objects anyway).
+int const qtextlayout_cache_max_size = 500;
 #else
-int cache_metrics_qtextlayout_size = 0;
+// Disable the cache
+int const qtextlayout_cache_max_size = 0;
 #endif
 
 
-namespace {
 /**
  * Convert a UCS4 character into a QChar.
  * This is a hack (it does only make sense for the common part of the UCS4
@@ -128,9 +130,9 @@ inline QChar const ucs4_to_qchar(char_type const ucs4)
 
 GuiFontMetrics::GuiFontMetrics(QFont const & font)
        : font_(font), metrics_(font, 0),
-         strwidth_cache_(cache_metrics_width_size),
-         breakstr_cache_(cache_metrics_breakstr_size),
-         qtextlayout_cache_(cache_metrics_qtextlayout_size)
+         strwidth_cache_(strwidth_cache_max_cost),
+         breakstr_cache_(breakstr_cache_max_cost),
+         qtextlayout_cache_(qtextlayout_cache_max_size)
 {
        // Determine italic slope
        double const defaultSlope = tan(qDegreesToRadians(19.0));

commit 792fdedc6c60d94092e21a924f40cde22a194fa1
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Wed Sep 22 15:09:26 2021 +0200

    Improve row flushing
    
    Add new row flags Flush and FlushBefore to let insets indicate whether
    they cause flushing of current row (eg. newline) or of previous row
    (e.g. display insets).

diff --git a/src/RowFlags.h b/src/RowFlags.h
index f94f0c6..01c4dd2 100644
--- a/src/RowFlags.h
+++ b/src/RowFlags.h
@@ -32,22 +32,26 @@ enum RowFlags {
        BreakBefore = 1 << 0,
        // Avoid breaking row before this element
        NoBreakBefore = 1 << 1,
+       // flush the row before this element (useful with BreakBefore)
+       FlushBefore = 1 << 2,
        // force new (maybe empty) row after this element
-       AlwaysBreakAfter = 1 << 2,
+       AlwaysBreakAfter = 1 << 3,
        // break row after this element if there are more elements
-       BreakAfter = 1 << 3,
+       BreakAfter = 1 << 4,
        // break row whenever needed after this element
-       CanBreakAfter = 1 << 4,
+       CanBreakAfter = 1 << 5,
        // Avoid breaking row after this element
-       NoBreakAfter = 1 << 5,
+       NoBreakAfter = 1 << 6,
        // The contents of the row may be broken in two (e.g. string)
-       CanBreakInside = 1 << 6,
+       CanBreakInside = 1 << 7,
+       // Flush the row that ends with this element
+       Flush = 1 << 8,
        // specify an alignment (left, right) for a display element
        // (default is center)
-       AlignLeft = 1 << 7,
-       AlignRight = 1 << 8,
+       AlignLeft = 1 << 9,
+       AlignRight = 1 << 10,
        // A display element breaks row at both ends
-       Display = BreakBefore | BreakAfter,
+       Display = FlushBefore | BreakBefore | BreakAfter,
        // Flags that concern breaking after element
        AfterFlags = AlwaysBreakAfter | BreakAfter | CanBreakAfter | 
NoBreakAfter
 };
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index 14dc705..2b1ee5a 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -1068,7 +1068,6 @@ void cleanupRow(Row & row, bool at_end)
        }
 
        row.endpos(row.back().endpos);
-       row.flushed(at_end);
        // remove trailing spaces on row break
        if (!at_end)
                row.back().rtrim();
@@ -1118,8 +1117,11 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
                int const f2 = (fcit == end) ? (end_label ? Inline : 
NoBreakBefore)
                                             : fcit->row_flags;
                if (rows.empty() || needsRowBreak(f1, f2)) {
-                       if (!rows.empty())
+                       if (!rows.empty()) {
                                cleanupRow(rows.back(), false);
+                               // Flush row as requested by row flags
+                               rows.back().flushed((f1 & Flush) || (f2 & 
FlushBefore));
+                       }
                        pos_type pos = rows.empty() ? 0 : rows.back().endpos();
                        rows.push_back(newRow(*this, bigrow.pit(), pos, 
is_rtl));
                        // the width available for the row.
diff --git a/src/insets/InsetNewline.cpp b/src/insets/InsetNewline.cpp
index f5fcd42..ecfacf4 100644
--- a/src/insets/InsetNewline.cpp
+++ b/src/insets/InsetNewline.cpp
@@ -39,6 +39,15 @@ InsetNewline::InsetNewline() : Inset(nullptr)
 {}
 
 
+int InsetNewline::rowFlags() const
+{
+       if (params_.kind == InsetNewlineParams::LINEBREAK)
+               return AlwaysBreakAfter;
+       else
+           return AlwaysBreakAfter | Flush;
+}
+
+
 void InsetNewlineParams::write(ostream & os) const
 {
        switch (kind) {
diff --git a/src/insets/InsetNewline.h b/src/insets/InsetNewline.h
index 1ef0ae5..c85a97d 100644
--- a/src/insets/InsetNewline.h
+++ b/src/insets/InsetNewline.h
@@ -47,7 +47,7 @@ public:
        explicit InsetNewline(InsetNewlineParams par) : Inset(0)
        { params_.kind = par.kind; }
        ///
-       int rowFlags() const override { return AlwaysBreakAfter; }
+       int rowFlags() const override;
        ///
        static void string2params(std::string const &, InsetNewlineParams &);
        ///
diff --git a/src/insets/InsetSeparator.h b/src/insets/InsetSeparator.h
index 9352bdf..0c12d95 100644
--- a/src/insets/InsetSeparator.h
+++ b/src/insets/InsetSeparator.h
@@ -65,7 +65,7 @@ public:
                return docstring();
        }
        ///
-       int rowFlags() const override { return BreakAfter; }
+       int rowFlags() const override { return BreakAfter | Flush; }
 private:
        ///
        InsetCode lyxCode() const override { return SEPARATOR_CODE; }

commit 27d413da25eefd7f58fbb0fd6ba9530616f79f71
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Wed Sep 22 13:17:46 2021 +0200

    Simplify setting of RTL in rows
    
    Set RTL status at row creation, which allows to remove a parameter from
    cleanupRow.

diff --git a/src/Row.cpp b/src/Row.cpp
index 6b0faa3..4b9836e 100644
--- a/src/Row.cpp
+++ b/src/Row.cpp
@@ -624,7 +624,7 @@ Row::Elements Row::shortenIfNeeded(int const w, int const 
next_width)
 }
 
 
-void Row::reverseRTL(bool const rtl_par)
+void Row::reverseRTL()
 {
        pos_type i = 0;
        pos_type const end = elements_.size();
@@ -636,14 +636,13 @@ void Row::reverseRTL(bool const rtl_par)
                        ++j;
                // if the direction is not the same as the paragraph
                // direction, the sequence has to be reverted.
-               if (rtl != rtl_par)
+               if (rtl != rtl_)
                        reverse(elements_.begin() + i, elements_.begin() + j);
                i = j;
        }
        // If the paragraph itself is RTL, reverse everything
-       if (rtl_par)
+       if (rtl_)
                reverse(elements_.begin(), elements_.end());
-       rtl_ = rtl_par;
 }
 
 Row::const_iterator const
diff --git a/src/Row.h b/src/Row.h
index bf49eb1..2c60638 100644
--- a/src/Row.h
+++ b/src/Row.h
@@ -301,10 +301,12 @@ public:
         * Find sequences of right-to-left elements and reverse them.
         * This should be called once the row is completely built.
         */
-       void reverseRTL(bool rtl_par);
+       void reverseRTL();
        ///
        bool isRTL() const { return rtl_; }
        ///
+       void setRTL(bool rtl) { rtl_ = rtl; }
+       ///
        bool needsChangeBar() const { return changebar_; }
        ///
        void needsChangeBar(bool ncb) { changebar_ = ncb; }
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index 76e2fb1..14dc705 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -1050,6 +1050,7 @@ Row newRow(TextMetrics const & tm, pit_type pit, pos_type 
pos, bool is_rtl)
        nrow.pos(pos);
        nrow.left_margin = tm.leftMargin(pit, pos);
        nrow.right_margin = tm.rightMargin(pit);
+       nrow.setRTL(is_rtl);
        if (is_rtl)
                swap(nrow.left_margin, nrow.right_margin);
        // Remember that the row width takes into account the left_margin
@@ -1059,7 +1060,7 @@ Row newRow(TextMetrics const & tm, pit_type pit, pos_type 
pos, bool is_rtl)
 }
 
 
-void cleanupRow(Row & row, bool at_end, bool is_rtl)
+void cleanupRow(Row & row, bool at_end)
 {
        if (row.empty()) {
                row.endpos(0);
@@ -1074,7 +1075,7 @@ void cleanupRow(Row & row, bool at_end, bool is_rtl)
        // boundary exists when there was no space at the end of row
        row.right_boundary(!at_end && row.back().endpos == row.endpos());
        // make sure that the RTL elements are in reverse ordering
-       row.reverseRTL(is_rtl);
+       row.reverseRTL();
 }
 
 
@@ -1118,7 +1119,7 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
                                             : fcit->row_flags;
                if (rows.empty() || needsRowBreak(f1, f2)) {
                        if (!rows.empty())
-                               cleanupRow(rows.back(), false, is_rtl);
+                               cleanupRow(rows.back(), false);
                        pos_type pos = rows.empty() ? 0 : rows.back().endpos();
                        rows.push_back(newRow(*this, bigrow.pit(), pos, 
is_rtl));
                        // the width available for the row.
@@ -1152,7 +1153,7 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
        }
 
        if (!rows.empty()) {
-               cleanupRow(rows.back(), true, is_rtl);
+               cleanupRow(rows.back(), true);
                // Last row in paragraph is flushed
                rows.back().flushed(true);
        }

commit 0ca1839154bb1b23cf31d281c9d752e4678922ad
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Mon Sep 6 14:52:42 2021 +0200

    Break multi-row strings in one pass
    
    Replace FontMetrics::breakAt, which returned the next break point,
    with FontMetrics::breakString, which returns a vector of break points.
    To this end, an additional parameter gives the available width for
    next rows.
    
    Rename various variables and methods accordingly. Factor the code in
    breakString_helper to be more manageable.
    
    Adapt Row::Element::splitAt to return a bool on sucess and provide
    remaining row elements in a vector. The width noted above has been
    added as parameters.
    
    Rename the helper function splitFrom to moveElements and rewrite the
    code to be more efficient.
    
    Remove type of row element INVALID, which is not needed anymore.
    
    The code in TextMetrics::breakParagraph is now much simpler.
    
    In Row::finalize, remove the code that computed inconditionnally the
    current element size, and make sure that this width will be computed
    in all code paths of Row::Element::splitAt.

diff --git a/src/Row.cpp b/src/Row.cpp
index b7e4c07..6b0faa3 100644
--- a/src/Row.cpp
+++ b/src/Row.cpp
@@ -123,41 +123,81 @@ pos_type Row::Element::x2pos(int &x) const
                        x = 0;
                        i = isRTL();
                }
-               break;
-       case INVALID:
-               LYXERR0("x2pos: INVALID row element !");
        }
        //lyxerr << "=> p=" << pos + i << " x=" << x << endl;
        return pos + i;
 }
 
 
-Row::Element Row::Element::splitAt(int w, bool force)
+bool Row::Element::splitAt(int const width, int next_width, bool force,
+                           Row::Elements & tail)
 {
-       if (type != STRING || !(row_flags & CanBreakInside))
-               return Element();
+       // Not a string or already OK.
+       if (type != STRING || (dim.wid > 0 && dim.wid < width))
+               return false;
 
        FontMetrics const & fm = theFontMetrics(font);
-       dim.wid = w;
-       int const i = fm.breakAt(str, dim.wid, isRTL(), force);
-       if (i != -1) {
-               //Create a second row element to return
-               Element ret(STRING, pos + i, font, change);
-               ret.str = str.substr(i);
-               ret.endpos = ret.pos + ret.str.length();
-               // Copy the after flags of the original element to the second 
one.
-               ret.row_flags = row_flags & (CanBreakInside | AfterFlags);
-
-               // Now update ourselves
-               str.erase(i);
-               endpos = pos + i;
-               // Row should be broken after the original element
-               row_flags = (row_flags & ~AfterFlags) | BreakAfter;
-               //LYXERR0("breakAt(" << w << ")  Row element Broken at " << w 
<< "(w(str)=" << fm.width(str) << "): e=" << *this);
-               return ret;
+
+       // A a string that is not breakable
+       if (!(row_flags & CanBreakInside)) {
+               // has width been computed yet?
+               if (dim.wid == 0)
+                       dim.wid = fm.width(str);
+               return false;
+       }
+
+       bool const wrap_any = !font.language()->wordWrap();
+       FontMetrics::Breaks breaks = fm.breakString(str, width, next_width,
+                                                isRTL(), wrap_any | force);
+
+       // if breaking did not really work, give up
+       if (!force && breaks.front().wid > width) {
+               if (dim.wid == 0)
+                       dim.wid = fm.width(str);
+               return false;
+       }
+
+       Element first_e(STRING, pos, font, change);
+       // should next element eventually replace *this?
+       bool first = true;
+       docstring::size_type i = 0;
+       for (FontMetrics::Break const & brk : breaks) {
+               Element e(STRING, pos + i, font, change);
+               e.str = str.substr(i, brk.len);
+               e.endpos = e.pos + brk.len;
+               e.dim.wid = brk.wid;
+               e.row_flags = CanBreakInside | BreakAfter;
+               if (first) {
+                       // this element eventually goes to *this
+                       e.row_flags |= row_flags & ~AfterFlags;
+                       first_e = e;
+                       first = false;
+               } else
+                       tail.push_back(e);
+               i += brk.len;
        }
 
-       return Element();
+       if (!tail.empty()) {
+               // Avoid having a last empty element. This happens when
+               // breaking at the trailing space of string
+               if (tail.back().str.empty())
+                       tail.pop_back();
+               else {
+                       // Copy the after flags of the original element to the 
last one.
+                       tail.back().row_flags &= ~BreakAfter;
+                       tail.back().row_flags |= row_flags & AfterFlags;
+               }
+               // first_e row should be broken after the original element
+               first_e.row_flags |= BreakAfter;
+       } else {
+               // Restore the after flags of the original element.
+               first_e.row_flags &= ~BreakAfter;
+               first_e.row_flags |= row_flags & AfterFlags;
+       }
+
+       // update ourselves
+       swap(first_e, *this);
+       return true;
 }
 
 
@@ -265,10 +305,6 @@ ostream & operator<<(ostream & os, Row::Element const & e)
                break;
        case Row::SPACE:
                os << "SPACE: ";
-               break;
-       case Row::INVALID:
-               os << "INVALID: ";
-               break;
        }
        os << "width=" << e.full_width() << ", row_flags=" << e.row_flags;
        return os;
@@ -393,11 +429,6 @@ void Row::finalizeLast()
        elt.final = true;
        if (elt.change.changed())
                changebar_ = true;
-
-       if (elt.type == STRING && elt.dim.wid == 0) {
-               elt.dim.wid = theFontMetrics(elt.font).width(elt.str);
-               dim_.wid += elt.dim.wid;
-       }
 }
 
 
@@ -474,19 +505,14 @@ void Row::pop_back()
 
 namespace {
 
-// Remove stuff after \c it from \c elts, and return it.
-// if \c init is provided, it will prepended to the rest
-Row::Elements splitFrom(Row::Elements & elts, Row::Elements::iterator const & 
it,
-                        Row::Element const & init = Row::Element())
+// Move stuff after \c it from \c from and the end of \c to.
+void moveElements(Row::Elements & from, Row::Elements::iterator const & it,
+                  Row::Elements & to)
 {
-       Row::Elements ret;
-       if (init.isValid())
-               ret.push_back(init);
-       ret.insert(ret.end(), it, elts.end());
-       elts.erase(it, elts.end());
-       if (!elts.empty())
-               elts.back().row_flags = (elts.back().row_flags & ~AfterFlags) | 
BreakAfter;
-       return ret;
+       to.insert(to.end(), it, from.end());
+       from.erase(it, from.end());
+       if (!from.empty())
+               from.back().row_flags = (from.back().row_flags & ~AfterFlags) | 
BreakAfter;
 }
 
 }
@@ -522,6 +548,7 @@ Row::Elements Row::shortenIfNeeded(int const w, int const 
next_width)
        Elements::iterator cit_brk = cit;
        int wid_brk = wid + cit_brk->dim.wid;
        ++cit_brk;
+       Elements tail;
        while (cit_brk != beg) {
                --cit_brk;
                // make a copy of the element to work on it.
@@ -533,28 +560,18 @@ Row::Elements Row::shortenIfNeeded(int const w, int const 
next_width)
                if (wid_brk <= w && brk.row_flags & CanBreakAfter) {
                        end_ = brk.endpos;
                        dim_.wid = wid_brk;
-                       return splitFrom(elements_, cit_brk + 1);
+                       moveElements(elements_, cit_brk + 1, tail);
+                       return tail;
                }
                // assume now that the current element is not there
                wid_brk -= brk.dim.wid;
-               /*
-                * Some Asian languages split lines anywhere (no notion of
-                * word). It seems that QTextLayout is not aware of this fact.
-                * See for reference:
-                *    
https://en.wikipedia.org/wiki/Line_breaking_rules_in_East_Asian_languages
-                *
-                * FIXME: Something shall be done about characters which are
-                * not allowed at the beginning or end of line.
-               */
-               bool const word_wrap = brk.font.language()->wordWrap();
                /* We have found a suitable separable element. This is the 
common case.
-                * Try to break it cleanly (at word boundary) at a length that 
is both
+                * Try to break it cleanly at a length that is both
                 * - less than the available space on the row
                 * - shorter than the natural width of the element, in order to 
enforce
                 *   break-up.
                 */
-               Element remainder = brk.splitAt(min(w - wid_brk, brk.dim.wid - 
2), !word_wrap);
-               if (brk.row_flags & BreakAfter) {
+               if (brk.splitAt(min(w - wid_brk, brk.dim.wid - 2), next_width, 
false, tail)) {
                        /* if this element originally did not cause a row 
overflow
                         * in itself, and the remainder of the row would still 
be
                         * too large after breaking, then we will have issues in
@@ -568,12 +585,10 @@ Row::Elements Row::shortenIfNeeded(int const w, int const 
next_width)
                        *cit_brk = brk;
                        dim_.wid = wid_brk + brk.dim.wid;
                        // If there are other elements, they should be removed.
-                       // remainder can be empty when splitting at trailing 
space
-                       if (remainder.str.empty())
-                               return splitFrom(elements_, next(cit_brk, 1));
-                       else
-                               return splitFrom(elements_, next(cit_brk, 1), 
remainder);
+                       moveElements(elements_, cit_brk + 1, tail);
+                       return tail;
                }
+               LATTEST(tail.empty());
        }
 
        if (cit != beg && cit->row_flags & NoBreakBefore) {
@@ -588,20 +603,23 @@ Row::Elements Row::shortenIfNeeded(int const w, int const 
next_width)
                // been added. We can cut right here.
                end_ = cit->pos;
                dim_.wid = wid;
-               return splitFrom(elements_, cit);
+               moveElements(elements_, cit, tail);
+               return tail;
        }
 
        /* If we are here, it means that we have not found a separator to
-        * shorten the row. Let's try to break it again, but not at word
-        * boundary this time.
+        * shorten the row. Let's try to break it again, but force
+        * splitting this time.
         */
-       Element remainder = cit->splitAt(w - wid, true);
-       if (cit->row_flags & BreakAfter) {
+       if (cit->splitAt(w - wid, next_width, true, tail)) {
+               LYXERR0(*cit);
                end_ = cit->endpos;
                dim_.wid = wid + cit->dim.wid;
                // If there are other elements, they should be removed.
-               return splitFrom(elements_, next(cit, 1), remainder);
+               moveElements(elements_, cit + 1, tail);
+               return tail;
        }
+
        return Elements();
 }
 
diff --git a/src/Row.h b/src/Row.h
index 1272a0f..bf49eb1 100644
--- a/src/Row.h
+++ b/src/Row.h
@@ -50,9 +50,7 @@ public:
                // An inset
                INSET,
                // Some spacing described by its width, not a string
-               SPACE,
-               // Something that should not happen (for error handling)
-               INVALID
+               SPACE
        };
 
 /**
@@ -61,8 +59,6 @@ public:
  */
        struct Element {
                //
-               Element() = default;
-               //
                Element(Type const t, pos_type p, Font const & f, Change const 
& ch)
                        : type(t), pos(p), endpos(p + 1), font(f), change(ch) {}
 
@@ -94,13 +90,16 @@ public:
                pos_type x2pos(int &x) const;
                /** Break the element in two if possible, so that its width is 
less
                 * than \param w.
-                * \return an element containing the remainder of the text, or
-                *   an invalid element if nothing happened.
-                * \param w: the desired maximum width
-                * \param force: if true, the string is cut at any place, 
otherwise it
-                *   respects the row breaking rules of characters.
+                * \return a vector of elements containing the remainder of
+                *   the text (empty if nothing happened).
+                * \param width maximum width of the row.
+                * \param next_width available width on next row.
+                * \param force: if true, cut string at any place, even for
+                *   languages that wrap at word delimiters; if false, do not
+                *   break at all if first element would larger than \c width.
                 */
-               Element splitAt(int w, bool force);
+               // FIXME: ideally last parameter should be Elements&, but it is 
not possible.
+               bool splitAt(int width, int next_width, bool force, 
std::vector<Element> & tail);
                // remove trailing spaces (useful for end of row)
                void rtrim();
 
@@ -108,8 +107,6 @@ public:
                bool isRTL() const { return font.isVisibleRightToLeft(); }
                // This is true for virtual elements.
                bool isVirtual() const { return type == VIRTUAL; }
-               // Invalid element, for error handling
-               bool isValid() const { return type !=INVALID; }
 
                // Returns the position on left side of the element.
                pos_type left_pos() const { return isRTL() ? endpos : pos; };
@@ -117,11 +114,11 @@ public:
                pos_type right_pos() const { return isRTL() ? pos : endpos; };
 
                // The kind of row element
-               Type type = INVALID;
+               Type type;
                // position of the element in the paragraph
-               pos_type pos = 0;
+               pos_type pos;
                // first position after the element in the paragraph
-               pos_type endpos = 0;
+               pos_type endpos;
                // The dimension of the chunk (does not contains the
                // separator correction)
                Dimension dim;
@@ -289,8 +286,8 @@ public:
         * separator and update endpos if necessary. If all that
         * remains is a large word, cut it to \param width.
         * \param width maximum width of the row.
-        * \param available width on next row.
-        * \return true if the row has been shortened.
+        * \param next_width available width on next row.
+        * \return list of elements remaining after breaking.
         */
        Elements shortenIfNeeded(int const width, int const next_width);
 
diff --git a/src/RowPainter.cpp b/src/RowPainter.cpp
index 656f89a..400b7b6 100644
--- a/src/RowPainter.cpp
+++ b/src/RowPainter.cpp
@@ -565,10 +565,6 @@ void RowPainter::paintText()
 
                case Row::SPACE:
                        paintTextDecoration(e);
-                       break;
-
-               case Row::INVALID:
-                       LYXERR0("Trying to paint INVALID row element.");
                }
 
                // The markings of foreign languages
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index d681a63..76e2fb1 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -1059,7 +1059,7 @@ Row newRow(TextMetrics const & tm, pit_type pit, pos_type 
pos, bool is_rtl)
 }
 
 
-void cleanupRow(Row & row, pos_type real_endpos, bool is_rtl)
+void cleanupRow(Row & row, bool at_end, bool is_rtl)
 {
        if (row.empty()) {
                row.endpos(0);
@@ -1067,11 +1067,12 @@ void cleanupRow(Row & row, pos_type real_endpos, bool 
is_rtl)
        }
 
        row.endpos(row.back().endpos);
+       row.flushed(at_end);
        // remove trailing spaces on row break
-       if (row.endpos() < real_endpos)
+       if (!at_end)
                row.back().rtrim();
        // boundary exists when there was no space at the end of row
-       row.right_boundary(row.endpos() < real_endpos && row.back().endpos == 
row.endpos());
+       row.right_boundary(!at_end && row.back().endpos == row.endpos());
        // make sure that the RTL elements are in reverse ordering
        row.reverseRTL(is_rtl);
 }
@@ -1098,6 +1099,8 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
        RowList rows;
        bool const is_rtl = text_->isRTL(bigrow.pit());
        bool const end_label = text_->getEndLabel(bigrow.pit()) != 
END_LABEL_NO_LABEL;
+       int const next_width = max_width_ - leftMargin(bigrow.pit(), 
bigrow.endpos())
+               - rightMargin(bigrow.pit());
 
        int width = 0;
        flexible_const_iterator<Row> fcit = flexible_begin(bigrow);
@@ -1115,7 +1118,7 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
                                             : fcit->row_flags;
                if (rows.empty() || needsRowBreak(f1, f2)) {
                        if (!rows.empty())
-                               cleanupRow(rows.back(), bigrow.endpos(), 
is_rtl);
+                               cleanupRow(rows.back(), false, is_rtl);
                        pos_type pos = rows.empty() ? 0 : rows.back().endpos();
                        rows.push_back(newRow(*this, bigrow.pit(), pos, 
is_rtl));
                        // the width available for the row.
@@ -1130,45 +1133,26 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
                // Next element to consider is either the top of the temporary
                // pile, or the place when we were in main row
                Row::Element elt = *fcit;
-               Row::Element next_elt = elt.splitAt(width - rows.back().width(),
-                                                   
!elt.font.language()->wordWrap());
-               if (elt.dim.wid > width - rows.back().width()) {
-                       Row & rb = rows.back();
-                       rb.push_back(*fcit);
-                       // if the row is too large, try to cut at last 
separator. In case
-                       // of success, reset indication that the row was broken 
abruptly.
-                       int const next_width = max_width_ - 
leftMargin(rb.pit(), rb.endpos())
-                               - rightMargin(rb.pit());
-
-                       Row::Elements next_elts = rb.shortenIfNeeded(width, 
next_width);
-
-                       // Go to next element
-                       ++fcit;
-
-                       // Handle later the elements returned by 
shortenIfNeeded.
-                       if (!next_elts.empty()) {
-                               rb.flushed(false);
-                               fcit.put(next_elts);
-                       }
-               } else {
-                       // a new element in the row
-                       rows.back().push_back(elt);
-                       rows.back().finalizeLast();
-
-                       // Go to next element
-                       ++fcit;
-
-                       // Add a new next element on the pile
-                       if (next_elt.isValid()) {
-                               // do as if we inserted this element in the 
original row
-                               if (!next_elt.str.empty())
-                                       fcit.put(next_elt);
-                       }
+               Row::Elements tail;
+               elt.splitAt(width - rows.back().width(), next_width, false, 
tail);
+               Row & rb = rows.back();
+               rb.push_back(elt);
+               rb.finalizeLast();
+               if (rb.width() > width) {
+                       LATTEST(tail.empty());
+                       // if the row is too large, try to cut at last 
separator.
+                       tail = rb.shortenIfNeeded(width, next_width);
                }
+
+               // Go to next element
+               ++fcit;
+
+               // Handle later the elements returned by splitAt or 
shortenIfNeeded.
+               fcit.put(tail);
        }
 
        if (!rows.empty()) {
-               cleanupRow(rows.back(), bigrow.endpos(), is_rtl);
+               cleanupRow(rows.back(), true, is_rtl);
                // Last row in paragraph is flushed
                rows.back().flushed(true);
        }
diff --git a/src/frontends/FontMetrics.h b/src/frontends/FontMetrics.h
index b562ebf..b78bc2d 100644
--- a/src/frontends/FontMetrics.h
+++ b/src/frontends/FontMetrics.h
@@ -16,6 +16,8 @@
 
 #include "support/strfwd.h"
 
+#include <vector>
+
 /**
  * A class holding helper functions for determining
  * the screen dimensions of fonts.
@@ -121,15 +123,27 @@ public:
         * \param ws is the amount of extra inter-word space applied text 
justification.
         */
        virtual int x2pos(docstring const & s, int & x, bool rtl, double ws) 
const = 0;
+
+       // The places where to break a string and the width of the resulting 
lines.
+       struct Break {
+               Break(int l, int w) : len(l), wid(w) {}
+               int len = 0;
+               int wid = 0;
+       };
+       typedef std::vector<Break> Breaks;
        /**
-        * Break string s at width at most x.
-        * \return break position (-1 if not successful)
-        * \param position x is updated to real width
-        * \param rtl is true for right-to-left layout
+        * Break a string in multiple fragments according to width limits.
+        * \return a sequence of Break elements.
+        * \param s is the string to break.
+        * \param first_wid is the available width for first line.
+        * \param wid is the available width for the next lines.
+        * \param rtl is true for right-to-left layout.
         * \param force is false for breaking at word separator, true for
         *   arbitrary position.
         */
-       virtual int breakAt(docstring const & s, int & x, bool rtl, bool force) 
const = 0;
+       virtual Breaks
+       breakString(docstring const & s, int first_wid, int wid, bool rtl, bool 
force) const = 0;
+
        /// return char dimension for the font.
        virtual Dimension const dimension(char_type c) const = 0;
        /**
diff --git a/src/frontends/qt/GuiFontMetrics.cpp 
b/src/frontends/qt/GuiFontMetrics.cpp
index 47537ae..3feb33e 100644
--- a/src/frontends/qt/GuiFontMetrics.cpp
+++ b/src/frontends/qt/GuiFontMetrics.cpp
@@ -19,6 +19,7 @@
 
 #include "support/convert.h"
 #include "support/lassert.h"
+#include "support/lstrings.h"
 #include "support/lyxlib.h"
 #include "support/debug.h"
 
@@ -86,14 +87,14 @@ namespace frontend {
 
 
 /*
- * Limit (strwidth|breakat)_cache_ size to 512kB of string data.
+ * Limit (strwidth|breakstr)_cache_ size to 512kB of string data.
  * Limit qtextlayout_cache_ size to 500 elements (we do not know the
  * size of the QTextLayout objects anyway).
  * Note that all these numbers are arbitrary.
  * Also, setting size to 0 is tantamount to disabling the cache.
  */
 int cache_metrics_width_size = 1 << 19;
-int cache_metrics_breakat_size = 1 << 19;
+int cache_metrics_breakstr_size = 1 << 19;
 // Qt 5.x already has its own caching of QTextLayout objects
 // but it does not seem to work well on MacOS X.
 #if (QT_VERSION < 0x050000) || defined(Q_OS_MAC)
@@ -128,7 +129,7 @@ inline QChar const ucs4_to_qchar(char_type const ucs4)
 GuiFontMetrics::GuiFontMetrics(QFont const & font)
        : font_(font), metrics_(font, 0),
          strwidth_cache_(cache_metrics_width_size),
-         breakat_cache_(cache_metrics_breakat_size),
+         breakstr_cache_(cache_metrics_breakstr_size),
          qtextlayout_cache_(cache_metrics_qtextlayout_size)
 {
        // Determine italic slope
@@ -485,11 +486,13 @@ int GuiFontMetrics::countExpanders(docstring const & str) 
const
 }
 
 
-pair<int, int>
-GuiFontMetrics::breakAt_helper(docstring const & s, int const x,
-                               bool const rtl, bool const force) const
+namespace {
+
+const int brkStrOffset = 1 + BIDI_OFFSET;
+
+
+QString createBreakableString(docstring const & s, bool rtl, QTextLayout & tl)
 {
-       QTextLayout tl;
        /* Qt will not break at a leading or trailing space, and we need
         * that sometimes, see http://www.lyx.org/trac/ticket/9921.
         *
@@ -518,34 +521,23 @@ GuiFontMetrics::breakAt_helper(docstring const & s, int 
const x,
                // Left-to-right override: forces to draw text left-to-right
                qs =  QChar(0x202D) + qs;
 #endif
-       int const offset = 1 + BIDI_OFFSET;
+       return qs;
+}
 
-       tl.setText(qs);
-       tl.setFont(font_);
-       QTextOption to;
-       to.setWrapMode(force ? QTextOption::WrapAtWordBoundaryOrAnywhere
-                            : QTextOption::WordWrap);
-       // Let QTextLine::naturalTextWidth() account for trailing spaces
-       // (horizontalAdvance() still does not).
-       to.setFlags(QTextOption::IncludeTrailingSpaces);
-       tl.setTextOption(to);
-       tl.beginLayout();
-       QTextLine line = tl.createLine();
-       line.setLineWidth(x);
-       tl.createLine();
-       tl.endLayout();
-       int line_wid = iround(line.horizontalAdvance());
-       if ((force && line.textLength() == offset) || line_wid > x)
-               return {-1, line_wid};
+
+docstring::size_type brkstr2str_pos(QString brkstr, docstring const & str, int 
pos)
+{
        /* Since QString is UTF-16 and docstring is UCS-4, the offsets may
         * not be the same when there are high-plan unicode characters
         * (bug #10443).
         */
-       // The variable `offset' is here to account for the extra leading 
characters.
+       // The variable `brkStrOffset' is here to account for the extra leading 
characters.
        // The ending character zerow_nbsp has to be ignored if the line is 
complete.
-       int const qlen = line.textLength() - offset - (line.textLength() == 
qs.length());
+       int const qlen = pos - brkStrOffset - (pos == brkstr.length());
 #if QT_VERSION < 0x040801 || QT_VERSION >= 0x050100
-       int len = qstring_to_ucs4(qs.mid(offset, qlen)).length();
+       auto const len = qstring_to_ucs4(brkstr.mid(brkStrOffset, 
qlen)).length();
+       // Avoid warning
+       (void)str;
 #else
        /* Due to QTBUG-25536 in 4.8.1 <= Qt < 5.1.0, the string returned
         * by QString::toUcs4 (used by qstring_to_ucs4) may have wrong
@@ -555,52 +547,108 @@ GuiFontMetrics::breakAt_helper(docstring const & s, int 
const x,
         * worthwhile to implement a dichotomy search if this shows up
         * under a profiler.
         */
-       int len = min(qlen, static_cast<int>(s.length()));
-       while (len >= 0 && toqstr(s.substr(0, len)).length() != qlen)
+       int len = min(qlen, static_cast<int>(str.length()));
+       while (len >= 0 && toqstr(str.substr(0, len)).length() != qlen)
                --len;
        LASSERT(len > 0 || qlen == 0, /**/);
 #endif
-       // Do not cut is the string is already short enough. We rely on
-       // naturalTextWidth() to catch the case where we cut at the trailing
-       // space.
-       if (len == static_cast<int>(s.length())
-               && line.naturalTextWidth() <= x) {
-               len = -1;
-#if QT_VERSION < 0x050000
+       return len;
+}
+
+}
+
+FontMetrics::Breaks
+GuiFontMetrics::breakString_helper(docstring const & s, int first_wid, int wid,
+                                   bool rtl, bool force) const
+{
+       QTextLayout tl;
+       QString qs = createBreakableString(s, rtl, tl);
+       tl.setText(qs);
+       tl.setFont(font_);
+       QTextOption to;
+       /*
+        * Some Asian languages split lines anywhere (no notion of
+        * word). It seems that QTextLayout is not aware of this fact.
+        * See for reference:
+        *    
https://en.wikipedia.org/wiki/Line_breaking_rules_in_East_Asian_languages
+        *
+        * FIXME: Something shall be done about characters which are
+        * not allowed at the beginning or end of line.
+        */
+       to.setWrapMode(force ? QTextOption::WrapAtWordBoundaryOrAnywhere
+                            : QTextOption::WordWrap);
+       // Let QTextLine::naturalTextWidth() account for trailing spaces
+       // (horizontalAdvance() still does not).
+       to.setFlags(QTextOption::IncludeTrailingSpaces);
+       tl.setTextOption(to);
+
+       bool first = true;
+       tl.beginLayout();
+       while(true) {
+               QTextLine line = tl.createLine();
+               if (!line.isValid())
+                       break;
+               line.setLineWidth(first ? first_wid : wid);
+               tl.createLine();
+               first = false;
+       }
+       tl.endLayout();
+
+       Breaks breaks;
+       int pos = 0;
+       for (int i = 0 ; i < tl.lineCount() ; ++i) {
+               QTextLine const & line = tl.lineAt(i);
+               int const epos = brkstr2str_pos(qs, s, line.textStart() + 
line.textLength());
+#if QT_VERSION >= 0x050000
+               int const wid = i + 1 < tl.lineCount() ? 
iround(line.horizontalAdvance())
+                                                      : 
iround(line.naturalTextWidth());
+#else
                // With some monospace fonts, the value of horizontalAdvance()
                // can be wrong with Qt4. One hypothesis is that the invisible
                // characters that we use are given a non-null width.
-               line_wid = width(s);
+               // FIXME: this is slower than it could be but we'll get rid of 
Qt4 anyway
+               int const wid = i + 1 < tl.lineCount() ? 
width(rtrim(s.substr(pos, epos - pos)))
+                                                      : width(s.substr(pos, 
epos - pos));
+#endif
+               breaks.emplace_back(epos - pos, wid);
+               pos = epos;
+#if 0
+               // FIXME: should it be kept in some form?
+               if ((force && line.textLength() == brkStrOffset) || line_wid > 
x)
+                       return {-1, line_wid};
 #endif
+
        }
-       return {len, line_wid};
+
+       return breaks;
 }
 
 
-uint qHash(BreakAtKey const & key)
+uint qHash(BreakStringKey const & key)
 {
-       int params = key.force + 2 * key.rtl + 4 * key.x;
+       // assume widths are less than 10000. This fits in 32 bits.
+       uint params = key.force + 2 * key.rtl + 4 * key.first_wid + 10000 * 
key.wid;
        return std::qHash(key.s) ^ ::qHash(params);
 }
 
 
-int GuiFontMetrics::breakAt(docstring const & s, int & x, bool const rtl, bool 
const force) const
+FontMetrics::Breaks GuiFontMetrics::breakString(docstring const & s, int 
first_wid, int wid,
+                                                bool rtl, bool force) const
 {
-       PROFILE_THIS_BLOCK(breakAt);
+       PROFILE_THIS_BLOCK(breakString);
        if (s.empty())
-               return false;
+               return Breaks();
 
-       BreakAtKey key{s, x, rtl, force};
-       pair<int, int> pp;
-       if (auto * pp_ptr = breakat_cache_.object_ptr(key))
-               pp = *pp_ptr;
+       BreakStringKey key{s, first_wid, wid, rtl, force};
+       Breaks brks;
+       if (auto * brks_ptr = breakstr_cache_.object_ptr(key))
+               brks = *brks_ptr;
        else {
-               PROFILE_CACHE_MISS(breakAt);
-               pp = breakAt_helper(s, x, rtl, force);
-               breakat_cache_.insert(key, pp, sizeof(key) + s.size() * 
sizeof(char_type));
+               PROFILE_CACHE_MISS(breakString);
+               brks = breakString_helper(s, first_wid, wid, rtl, force);
+               breakstr_cache_.insert(key, brks, sizeof(key) + s.size() * 
sizeof(char_type));
        }
-       x = pp.second;
-       return pp.first;
+       return brks;
 }
 
 
diff --git a/src/frontends/qt/GuiFontMetrics.h 
b/src/frontends/qt/GuiFontMetrics.h
index ef8588a..9501eb8 100644
--- a/src/frontends/qt/GuiFontMetrics.h
+++ b/src/frontends/qt/GuiFontMetrics.h
@@ -27,14 +27,16 @@
 namespace lyx {
 namespace frontend {
 
-struct BreakAtKey
+struct BreakStringKey
 {
-       bool operator==(BreakAtKey const & key) const {
-               return key.s == s && key.x == x && key.rtl == rtl && key.force 
== force;
+       bool operator==(BreakStringKey const & key) const {
+               return key.s == s && key.first_wid == first_wid && key.wid == 
wid
+                       && key.rtl == rtl && key.force == force;
        }
 
        docstring s;
-       int x;
+       int first_wid;
+       int wid;
        bool rtl;
        bool force;
 };
@@ -77,7 +79,7 @@ public:
        int signedWidth(docstring const & s) const override;
        int pos2x(docstring const & s, int pos, bool rtl, double ws) const 
override;
        int x2pos(docstring const & s, int & x, bool rtl, double ws) const 
override;
-       int breakAt(docstring const & s, int & x, bool rtl, bool force) const 
override;
+       Breaks breakString(docstring const & s, int first_wid, int wid, bool 
rtl, bool force) const override;
        Dimension const dimension(char_type c) const override;
 
        void rectText(docstring const & str,
@@ -101,8 +103,8 @@ public:
 
 private:
 
-       std::pair<int, int> breakAt_helper(docstring const & s, int const x,
-                                          bool const rtl, bool const force) 
const;
+       Breaks breakString_helper(docstring const & s, int first_wid, int wid,
+                                 bool rtl, bool force) const;
 
        /// The font
        QFont font_;
@@ -117,8 +119,8 @@ private:
        mutable QHash<char_type, int> width_cache_;
        /// Cache of string widths
        mutable Cache<docstring, int> strwidth_cache_;
-       /// Cache for breakAt
-       mutable Cache<BreakAtKey, std::pair<int, int>> breakat_cache_;
+       /// Cache for breakString
+       mutable Cache<BreakStringKey, Breaks> breakstr_cache_;
        /// Cache for QTextLayout
        mutable Cache<TextLayoutKey, std::shared_ptr<QTextLayout>> 
qtextlayout_cache_;
 

commit b659bb1b95878184dfd49f12f1907092b149b326
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Mon Sep 20 17:32:18 2021 +0200

    Add operator<< for Row::Elements
    
    This is useful for debugging.

diff --git a/src/Row.cpp b/src/Row.cpp
index 8233a96..b7e4c07 100644
--- a/src/Row.cpp
+++ b/src/Row.cpp
@@ -275,6 +275,17 @@ ostream & operator<<(ostream & os, Row::Element const & e)
 }
 
 
+ostream & operator<<(ostream & os, Row::Elements const & elts)
+{
+       double x = 0;
+       for (Row::Element const & e : elts) {
+               os << "x=" << x << " => " << e << endl;
+               x += e.full_width();
+       }
+       return os;
+}
+
+
 ostream & operator<<(ostream & os, Row const & row)
 {
        os << " pos: " << row.pos_ << " end: " << row.end_
@@ -286,11 +297,11 @@ ostream & operator<<(ostream & os, Row const & row)
           << " separator: " << row.separator
           << " label_hfill: " << row.label_hfill
           << " row_boundary: " << row.right_boundary() << "\n";
+       // We cannot use the operator above, unfortunately
        double x = row.left_margin;
-       Row::Elements::const_iterator it = row.elements_.begin();
-       for ( ; it != row.elements_.end() ; ++it) {
-               os << "x=" << x << " => " << *it << endl;
-               x += it->full_width();
+       for (Row::Element const & e : row.elements_) {
+               os << "x=" << x << " => " << e << endl;
+               x += e.full_width();
        }
        return os;
 }
diff --git a/src/Row.h b/src/Row.h
index 7466797..1272a0f 100644
--- a/src/Row.h
+++ b/src/Row.h
@@ -378,6 +378,8 @@ private:
        bool changebar_ = false;
 };
 
+std::ostream & operator<<(std::ostream & os, Row::Elements const & elts);
+
 
 /**
  * Each paragraph is broken up into a number of rows on the screen.

commit 2f70061dae2ce2bfd7bf5bd521bde15afe441f58
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Thu Sep 2 15:16:28 2021 +0200

    Fix setting of row pos/endpos (overlapping rows)
    
    In TextMetrics::breakParagraph, get rid of the fragile `pos' local
    variable, which was not correctly updated. Rely on the endpos of the
    last element in row instead.
    
    Rewrite cleanupRow to rely on the endpos of last the row element to
    set row endpos, instead of a `pos' parameter.

diff --git a/src/Row.cpp b/src/Row.cpp
index 7b74694..8233a96 100644
--- a/src/Row.cpp
+++ b/src/Row.cpp
@@ -463,8 +463,8 @@ void Row::pop_back()
 
 namespace {
 
-// Remove stuff after it from elts, and return it.
-// if init is provided, it will be in front of the rest
+// Remove stuff after \c it from \c elts, and return it.
+// if \c init is provided, it will prepended to the rest
 Row::Elements splitFrom(Row::Elements & elts, Row::Elements::iterator const & 
it,
                         Row::Element const & init = Row::Element())
 {
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index 8812daf..d681a63 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -1059,15 +1059,19 @@ Row newRow(TextMetrics const & tm, pit_type pit, 
pos_type pos, bool is_rtl)
 }
 
 
-void cleanupRow(Row & row, pos_type pos, pos_type real_endpos, bool is_rtl)
+void cleanupRow(Row & row, pos_type real_endpos, bool is_rtl)
 {
-       row.endpos(pos);
+       if (row.empty()) {
+               row.endpos(0);
+               return;
+       }
+
+       row.endpos(row.back().endpos);
        // remove trailing spaces on row break
-       if (pos < real_endpos && !row.empty())
+       if (row.endpos() < real_endpos)
                row.back().rtrim();
        // boundary exists when there was no space at the end of row
-       row.right_boundary(!row.empty() && pos < real_endpos
-                          && row.back().endpos == pos);
+       row.right_boundary(row.endpos() < real_endpos && row.back().endpos == 
row.endpos());
        // make sure that the RTL elements are in reverse ordering
        row.reverseRTL(is_rtl);
 }
@@ -1095,7 +1099,6 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
        bool const is_rtl = text_->isRTL(bigrow.pit());
        bool const end_label = text_->getEndLabel(bigrow.pit()) != 
END_LABEL_NO_LABEL;
 
-       pos_type pos = 0;
        int width = 0;
        flexible_const_iterator<Row> fcit = flexible_begin(bigrow);
        flexible_const_iterator<Row> const end = flexible_end(bigrow);
@@ -1112,7 +1115,8 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
                                             : fcit->row_flags;
                if (rows.empty() || needsRowBreak(f1, f2)) {
                        if (!rows.empty())
-                               cleanupRow(rows.back(), pos, bigrow.endpos(), 
is_rtl);
+                               cleanupRow(rows.back(), bigrow.endpos(), 
is_rtl);
+                       pos_type pos = rows.empty() ? 0 : rows.back().endpos();
                        rows.push_back(newRow(*this, bigrow.pit(), pos, 
is_rtl));
                        // the width available for the row.
                        width = max_width_ - rows.back().right_margin;
@@ -1150,7 +1154,6 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
                        // a new element in the row
                        rows.back().push_back(elt);
                        rows.back().finalizeLast();
-                       pos = elt.endpos;
 
                        // Go to next element
                        ++fcit;
@@ -1165,7 +1168,7 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
        }
 
        if (!rows.empty()) {
-               cleanupRow(rows.back(), pos, bigrow.endpos(), is_rtl);
+               cleanupRow(rows.back(), bigrow.endpos(), is_rtl);
                // Last row in paragraph is flushed
                rows.back().flushed(true);
        }

commit d672853553383be0767ac4d33088d83206da7f17
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Wed Sep 1 16:54:28 2021 +0200

    Get rid of need_new_row boolean in breakParagraph
    
    Instead of having breakParagraph decide when breaking a row is
    necessary, let Row::shortenIfNeeded set the row_flag of the last
    element to request a row break. This was already done in splitAt.
    
    This is in preparation of splitAt splitting in more than two elements.

diff --git a/src/Row.cpp b/src/Row.cpp
index d3529a3..7b74694 100644
--- a/src/Row.cpp
+++ b/src/Row.cpp
@@ -473,6 +473,8 @@ Row::Elements splitFrom(Row::Elements & elts, 
Row::Elements::iterator const & it
                ret.push_back(init);
        ret.insert(ret.end(), it, elts.end());
        elts.erase(it, elts.end());
+       if (!elts.empty())
+               elts.back().row_flags = (elts.back().row_flags & ~AfterFlags) | 
BreakAfter;
        return ret;
 }
 
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index a7dd300..8812daf 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -1042,6 +1042,7 @@ bool operator==(flexible_const_iterator<T> const & t1,
        return t1.cit_ == t2.cit_ && t1.pile_.empty() && t2.pile_.empty();
 }
 
+
 Row newRow(TextMetrics const & tm, pit_type pit, pos_type pos, bool is_rtl)
 {
        Row nrow;
@@ -1071,6 +1072,7 @@ void cleanupRow(Row & row, pos_type pos, pos_type 
real_endpos, bool is_rtl)
        row.reverseRTL(is_rtl);
 }
 
+
 // Implement the priorities described in RowFlags.h.
 bool needsRowBreak(int f1, int f2)
 {
@@ -1093,14 +1095,12 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
        bool const is_rtl = text_->isRTL(bigrow.pit());
        bool const end_label = text_->getEndLabel(bigrow.pit()) != 
END_LABEL_NO_LABEL;
 
-       bool need_new_row = true;
        pos_type pos = 0;
        int width = 0;
        flexible_const_iterator<Row> fcit = flexible_begin(bigrow);
        flexible_const_iterator<Row> const end = flexible_end(bigrow);
        while (true) {
-               bool const has_row = !rows.empty();
-               bool const row_empty = !has_row || rows.back().empty();
+               bool const row_empty = rows.empty() || rows.back().empty();
                // The row flags of previous element, if there is one.
                // Otherwise we use NoBreakAfter to avoid an empty row before
                // e.g. a displayed equation.
@@ -1110,14 +1110,12 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
                // paragraph has an end label (for which an empty row is OK).
                int const f2 = (fcit == end) ? (end_label ? Inline : 
NoBreakBefore)
                                             : fcit->row_flags;
-               need_new_row |= needsRowBreak(f1, f2);
-               if (need_new_row) {
+               if (rows.empty() || needsRowBreak(f1, f2)) {
                        if (!rows.empty())
                                cleanupRow(rows.back(), pos, bigrow.endpos(), 
is_rtl);
                        rows.push_back(newRow(*this, bigrow.pit(), pos, 
is_rtl));
                        // the width available for the row.
                        width = max_width_ - rows.back().right_margin;
-                       need_new_row = false;
                }
 
                // The stopping condition is here because we may need a new
@@ -1147,7 +1145,6 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
                        if (!next_elts.empty()) {
                                rb.flushed(false);
                                fcit.put(next_elts);
-                               need_new_row = true;
                        }
                } else {
                        // a new element in the row
@@ -1163,7 +1160,6 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
                                // do as if we inserted this element in the 
original row
                                if (!next_elt.str.empty())
                                        fcit.put(next_elt);
-                               need_new_row = true;
                        }
                }
        }

commit 3ffc1c091593771e7a54a0ed6e765231cf98cd74
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Tue Aug 31 19:23:55 2021 +0200

    Centralize the code that removes trailing spaces from end row element.
    
    Move to Row::Element::rtrim the code in Row::shortenIfNeeded that
    removes trailing spaces from last element in row, so that it can be
    called when actually breaking a row.
    
    Fixes bug found by Kornel.

diff --git a/src/Row.cpp b/src/Row.cpp
index 38dbb85..d3529a3 100644
--- a/src/Row.cpp
+++ b/src/Row.cpp
@@ -35,7 +35,6 @@ using namespace std;
 
 namespace lyx {
 
-using support::rtrim;
 using frontend::FontMetrics;
 
 
@@ -162,6 +161,20 @@ Row::Element Row::Element::splitAt(int w, bool force)
 }
 
 
+void Row::Element::rtrim()
+{
+       if (type != STRING)
+               return;
+       /* This is intended for strings that have been created by splitAt.
+        * They may have trailing spaces, but they are not counted in the
+        * string length (QTextLayout feature, actually). We remove them,
+        * and decrease endpos, since spaces at row break are invisible.
+        */
+       str = support::rtrim(str);
+       endpos = pos + str.length();
+}
+
+
 bool Row::isMarginSelected(bool left, DocIterator const & beg,
                DocIterator const & end) const
 {
@@ -539,14 +552,6 @@ Row::Elements Row::shortenIfNeeded(int const w, int const 
next_width)
                                break;
                        }
                        end_ = brk.endpos;
-                       /* after breakAt, there may be spaces at the end of the
-                        * string, but they are not counted in the string length
-                        * (QTextLayout feature, actually). We remove them, but 
do
-                        * not change the end of the row, since spaces at row
-                        * break are invisible.
-                        */
-                       brk.str = rtrim(brk.str);
-                       brk.endpos = brk.pos + brk.str.length();
                        *cit_brk = brk;
                        dim_.wid = wid_brk + brk.dim.wid;
                        // If there are other elements, they should be removed.
@@ -578,11 +583,8 @@ Row::Elements Row::shortenIfNeeded(int const w, int const 
next_width)
         * boundary this time.
         */
        Element remainder = cit->splitAt(w - wid, true);
-       if (remainder.isValid()) {
+       if (cit->row_flags & BreakAfter) {
                end_ = cit->endpos;
-               // See comment above.
-               cit->str = rtrim(cit->str);
-               cit->endpos = cit->pos + cit->str.length();
                dim_.wid = wid + cit->dim.wid;
                // If there are other elements, they should be removed.
                return splitFrom(elements_, next(cit, 1), remainder);
diff --git a/src/Row.h b/src/Row.h
index b0b1755..7466797 100644
--- a/src/Row.h
+++ b/src/Row.h
@@ -101,6 +101,8 @@ public:
                 *   respects the row breaking rules of characters.
                 */
                Element splitAt(int w, bool force);
+               // remove trailing spaces (useful for end of row)
+               void rtrim();
 
                //
                bool isRTL() const { return font.isVisibleRightToLeft(); }
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index 37894f6..a7dd300 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -1061,6 +1061,10 @@ Row newRow(TextMetrics const & tm, pit_type pit, 
pos_type pos, bool is_rtl)
 void cleanupRow(Row & row, pos_type pos, pos_type real_endpos, bool is_rtl)
 {
        row.endpos(pos);
+       // remove trailing spaces on row break
+       if (pos < real_endpos && !row.empty())
+               row.back().rtrim();
+       // boundary exists when there was no space at the end of row
        row.right_boundary(!row.empty() && pos < real_endpos
                           && row.back().endpos == pos);
        // make sure that the RTL elements are in reverse ordering

commit 9bcfa33fa692193dfc5f0bc14627dcb764ce98c6
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Tue Aug 31 15:58:56 2021 +0200

    Handle the case where breakAt cuts after trailing space
    
    In this case, the extra element returned should empty but valid. The
    row flag BreakAfter is set to indicate that we have a break there
    (this principle will be used more generally in a forthcoming commit).
    
    To detect that we cut at the trailing space, it is necessary to rely
    on the difference between QTextLine::horizontalAdvance() and
    QTextLine::naturalTextWidth() when the flag
    QTextOption::IncludeTrailingSpaces is used: the trailing space is
    taken into account in the later, but not in the former.
    
    Somme comments have been added to make code intent clearer.

diff --git a/src/Row.cpp b/src/Row.cpp
index 6e975a5..38dbb85 100644
--- a/src/Row.cpp
+++ b/src/Row.cpp
@@ -142,13 +142,19 @@ Row::Element Row::Element::splitAt(int w, bool force)
        dim.wid = w;
        int const i = fm.breakAt(str, dim.wid, isRTL(), force);
        if (i != -1) {
+               //Create a second row element to return
                Element ret(STRING, pos + i, font, change);
                ret.str = str.substr(i);
                ret.endpos = ret.pos + ret.str.length();
+               // Copy the after flags of the original element to the second 
one.
                ret.row_flags = row_flags & (CanBreakInside | AfterFlags);
+
+               // Now update ourselves
                str.erase(i);
                endpos = pos + i;
-               //lyxerr << "breakAt(" << w << ")  Row element Broken at " << x 
<< "(w(str)=" << fm.width(str) << "): e=" << *this << endl;
+               // Row should be broken after the original element
+               row_flags = (row_flags & ~AfterFlags) | BreakAfter;
+               //LYXERR0("breakAt(" << w << ")  Row element Broken at " << w 
<< "(w(str)=" << fm.width(str) << "): e=" << *this);
                return ret;
        }
 
@@ -251,7 +257,7 @@ ostream & operator<<(ostream & os, Row::Element const & e)
                os << "INVALID: ";
                break;
        }
-       os << "width=" << e.full_width();
+       os << "width=" << e.full_width() << ", row_flags=" << e.row_flags;
        return os;
 }
 
@@ -522,7 +528,7 @@ Row::Elements Row::shortenIfNeeded(int const w, int const 
next_width)
                 *   break-up.
                 */
                Element remainder = brk.splitAt(min(w - wid_brk, brk.dim.wid - 
2), !word_wrap);
-               if (remainder.isValid()) {
+               if (brk.row_flags & BreakAfter) {
                        /* if this element originally did not cause a row 
overflow
                         * in itself, and the remainder of the row would still 
be
                         * too large after breaking, then we will have issues in
@@ -544,7 +550,11 @@ Row::Elements Row::shortenIfNeeded(int const w, int const 
next_width)
                        *cit_brk = brk;
                        dim_.wid = wid_brk + brk.dim.wid;
                        // If there are other elements, they should be removed.
-                       return splitFrom(elements_, next(cit_brk, 1), 
remainder);
+                       // remainder can be empty when splitting at trailing 
space
+                       if (remainder.str.empty())
+                               return splitFrom(elements_, next(cit_brk, 1));
+                       else
+                               return splitFrom(elements_, next(cit_brk, 1), 
remainder);
                }
        }
 
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index c7e84f6..37894f6 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -1157,7 +1157,8 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
                        // Add a new next element on the pile
                        if (next_elt.isValid()) {
                                // do as if we inserted this element in the 
original row
-                               fcit.put(next_elt);
+                               if (!next_elt.str.empty())
+                                       fcit.put(next_elt);
                                need_new_row = true;
                        }
                }
diff --git a/src/frontends/qt/GuiFontMetrics.cpp 
b/src/frontends/qt/GuiFontMetrics.cpp
index 56ed676..47537ae 100644
--- a/src/frontends/qt/GuiFontMetrics.cpp
+++ b/src/frontends/qt/GuiFontMetrics.cpp
@@ -525,6 +525,9 @@ GuiFontMetrics::breakAt_helper(docstring const & s, int 
const x,
        QTextOption to;
        to.setWrapMode(force ? QTextOption::WrapAtWordBoundaryOrAnywhere
                             : QTextOption::WordWrap);
+       // Let QTextLine::naturalTextWidth() account for trailing spaces
+       // (horizontalAdvance() still does not).
+       to.setFlags(QTextOption::IncludeTrailingSpaces);
        tl.setTextOption(to);
        tl.beginLayout();
        QTextLine line = tl.createLine();
@@ -557,8 +560,11 @@ GuiFontMetrics::breakAt_helper(docstring const & s, int 
const x,
                --len;
        LASSERT(len > 0 || qlen == 0, /**/);
 #endif
-       // Do not cut is the string is already short enough
-       if (len == static_cast<int>(s.length())) {
+       // Do not cut is the string is already short enough. We rely on
+       // naturalTextWidth() to catch the case where we cut at the trailing
+       // space.
+       if (len == static_cast<int>(s.length())
+               && line.naturalTextWidth() <= x) {
                len = -1;
 #if QT_VERSION < 0x050000
                // With some monospace fonts, the value of horizontalAdvance()

commit 9b419e368d1c5fab0844ac2f01444ff3370daec5
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Mon Aug 30 15:48:44 2021 +0200

    Workaround for Qt 4
    
    At least with Qt 4.8.7 on Ubuntu 16.04, QTextLine::lineWidth() can
    return a bogus value, at least with Courier font. One hypothesis is
    that the invisible characters that we use in breakAt_helper are given
    a non-null width.
    
    Work around it, although the exact bug has not been pinpointed.

diff --git a/src/frontends/qt/GuiFontMetrics.cpp 
b/src/frontends/qt/GuiFontMetrics.cpp
index 25bf7a4..56ed676 100644
--- a/src/frontends/qt/GuiFontMetrics.cpp
+++ b/src/frontends/qt/GuiFontMetrics.cpp
@@ -531,7 +531,7 @@ GuiFontMetrics::breakAt_helper(docstring const & s, int 
const x,
        line.setLineWidth(x);
        tl.createLine();
        tl.endLayout();
-       int const line_wid = iround(line.horizontalAdvance());
+       int line_wid = iround(line.horizontalAdvance());
        if ((force && line.textLength() == offset) || line_wid > x)
                return {-1, line_wid};
        /* Since QString is UTF-16 and docstring is UCS-4, the offsets may
@@ -557,9 +557,16 @@ GuiFontMetrics::breakAt_helper(docstring const & s, int 
const x,
                --len;
        LASSERT(len > 0 || qlen == 0, /**/);
 #endif
-       // si la chaîne est déjà trop courte, on ne coupe pas
-       if (len == static_cast<int>(s.length()))
+       // Do not cut is the string is already short enough
+       if (len == static_cast<int>(s.length())) {
                len = -1;
+#if QT_VERSION < 0x050000
+               // With some monospace fonts, the value of horizontalAdvance()
+               // can be wrong with Qt4. One hypothesis is that the invisible
+               // characters that we use are given a non-null width.
+               line_wid = width(s);
+#endif
+       }
        return {len, line_wid};
 }
 

commit 60aff873397b84ea213eabdb38502fe484d1af2f
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Tue Jul 20 00:07:13 2021 +0200

    Last step of transition: use sortenIfNeeded again.
    
    Change semantics of Row::shortenIfNeeded: instead of breaking the row
    and returning a boolean, it returns the list of row elements that have
    been removed (or broken) from the row. The logic of the method remains
    the same.
    
    Use shortenIfNeeded in breakParagraph. This was the last missing block.
    
    Remove Row::breakAt and the old breakRow. Only bugs remain now :)

diff --git a/src/Row.cpp b/src/Row.cpp
index 3bc8ef0..6e975a5 100644
--- a/src/Row.cpp
+++ b/src/Row.cpp
@@ -156,12 +156,6 @@ Row::Element Row::Element::splitAt(int w, bool force)
 }
 
 
-bool Row::Element::breakAt(int w, bool force)
-{
-       return splitAt(w, force).isValid();
-}
-
-
 bool Row::isMarginSelected(bool left, DocIterator const & beg,
                DocIterator const & end) const
 {
@@ -448,10 +442,31 @@ void Row::pop_back()
 }
 
 
-bool Row::shortenIfNeeded(int const w, int const next_width)
+namespace {
+
+// Remove stuff after it from elts, and return it.
+// if init is provided, it will be in front of the rest
+Row::Elements splitFrom(Row::Elements & elts, Row::Elements::iterator const & 
it,
+                        Row::Element const & init = Row::Element())
+{
+       Row::Elements ret;
+       if (init.isValid())
+               ret.push_back(init);
+       ret.insert(ret.end(), it, elts.end());
+       elts.erase(it, elts.end());
+       return ret;
+}
+
+}
+
+
+Row::Elements Row::shortenIfNeeded(int const w, int const next_width)
 {
+       // FIXME: performance: if the last element is a string, we would
+       // like to avoid computing its length.
+       finalizeLast();
        if (empty() || width() <= w)
-               return false;
+               return Elements();
 
        Elements::iterator const beg = elements_.begin();
        Elements::iterator const end = elements_.end();
@@ -467,8 +482,8 @@ bool Row::shortenIfNeeded(int const w, int const next_width)
 
        if (cit == end) {
                // This should not happen since the row is too long.
-               LYXERR0("Something is wrong cannot shorten row: " << *this);
-               return false;
+               LYXERR0("Something is wrong, cannot shorten row: " << *this);
+               return Elements();
        }
 
        // Iterate backwards over breakable elements and try to break them
@@ -486,8 +501,7 @@ bool Row::shortenIfNeeded(int const w, int const next_width)
                if (wid_brk <= w && brk.row_flags & CanBreakAfter) {
                        end_ = brk.endpos;
                        dim_.wid = wid_brk;
-                       elements_.erase(cit_brk + 1, end);
-                       return true;
+                       return splitFrom(elements_, cit_brk + 1);
                }
                // assume now that the current element is not there
                wid_brk -= brk.dim.wid;
@@ -507,7 +521,8 @@ bool Row::shortenIfNeeded(int const w, int const next_width)
                 * - shorter than the natural width of the element, in order to 
enforce
                 *   break-up.
                 */
-               if (brk.breakAt(min(w - wid_brk, brk.dim.wid - 2), !word_wrap)) 
{
+               Element remainder = brk.splitAt(min(w - wid_brk, brk.dim.wid - 
2), !word_wrap);
+               if (remainder.isValid()) {
                        /* if this element originally did not cause a row 
overflow
                         * in itself, and the remainder of the row would still 
be
                         * too large after breaking, then we will have issues in
@@ -529,14 +544,13 @@ bool Row::shortenIfNeeded(int const w, int const 
next_width)
                        *cit_brk = brk;
                        dim_.wid = wid_brk + brk.dim.wid;
                        // If there are other elements, they should be removed.
-                       elements_.erase(cit_brk + 1, end);
-                       return true;
+                       return splitFrom(elements_, next(cit_brk, 1), 
remainder);
                }
        }
 
-       if (cit != beg && cit->type == VIRTUAL) {
-               // It is not possible to separate a virtual element from the
-               // previous one.
+       if (cit != beg && cit->row_flags & NoBreakBefore) {
+               // It is not possible to separate this element from the
+               // previous one. (e.g. VIRTUAL)
                --cit;
                wid -= cit->dim.wid;
        }
@@ -546,25 +560,24 @@ bool Row::shortenIfNeeded(int const w, int const 
next_width)
                // been added. We can cut right here.
                end_ = cit->pos;
                dim_.wid = wid;
-               elements_.erase(cit, end);
-               return true;
+               return splitFrom(elements_, cit);
        }
 
        /* If we are here, it means that we have not found a separator to
         * shorten the row. Let's try to break it again, but not at word
         * boundary this time.
         */
-       if (cit->breakAt(w - wid, true)) {
+       Element remainder = cit->splitAt(w - wid, true);
+       if (remainder.isValid()) {
                end_ = cit->endpos;
                // See comment above.
                cit->str = rtrim(cit->str);
                cit->endpos = cit->pos + cit->str.length();
                dim_.wid = wid + cit->dim.wid;
                // If there are other elements, they should be removed.
-               elements_.erase(next(cit, 1), end);
-               return true;
+               return splitFrom(elements_, next(cit, 1), remainder);
        }
-       return false;
+       return Elements();
 }
 
 
diff --git a/src/Row.h b/src/Row.h
index 72d7a86..b0b1755 100644
--- a/src/Row.h
+++ b/src/Row.h
@@ -101,12 +101,6 @@ public:
                 *   respects the row breaking rules of characters.
                 */
                Element splitAt(int w, bool force);
-               /** Break the element if possible, so that its width is less
-                * than \param w. Returns true on success. When \param force
-                * is true, the string is cut at any place, otherwise it
-                * respects the row breaking rules of characters.
-                */
-               bool breakAt(int w, bool force);
 
                //
                bool isRTL() const { return font.isVisibleRightToLeft(); }
@@ -296,7 +290,7 @@ public:
         * \param available width on next row.
         * \return true if the row has been shortened.
         */
-       bool shortenIfNeeded(int const width, int const next_width);
+       Elements shortenIfNeeded(int const width, int const next_width);
 
        /**
         * If last element of the row is a string, compute its width
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index 1a6927a..c7e84f6 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -491,7 +491,7 @@ bool TextMetrics::redoParagraph(pit_type const pit, bool 
const align_rows)
 
                // If there is an end of paragraph marker, its size should be
                // substracted to the available width. The logic here is
-               // almost the same as in breakRow, remember keep them in sync.
+               // almost the same as in tokenizeParagraph, remember keep them 
in sync.
                int eop = 0;
                if (e.pos + 1 == par.size()
                      && (lyxrc.paragraph_markers || 
par.lookupChange(par.size()).changed())
@@ -1006,6 +1006,11 @@ public:
 
        void put(value_type const & e) { pile_.push_back(e); }
 
+       // Put a sequence of elements on the pile (in reverse order!)
+       void put(vector<value_type> const & elts) {
+               pile_.insert(pile_.end(), elts.rbegin(), elts.rend());
+       }
+
 // This should be private, but declaring the friend functions is too much work
 //private:
        typename T::const_iterator cit_;
@@ -1121,19 +1126,40 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
                Row::Element elt = *fcit;
                Row::Element next_elt = elt.splitAt(width - rows.back().width(),
                                                    
!elt.font.language()->wordWrap());
-               // a new element in the row
-               rows.back().push_back(elt);
-               rows.back().finalizeLast();
-               pos = elt.endpos;
-
-               // Go to next element
-               ++fcit;
-
-               // Add a new next element on the pile
-               if (next_elt.isValid()) {
-                       // do as if we inserted this element in the original row
-                       fcit.put(next_elt);
-                       need_new_row = true;
+               if (elt.dim.wid > width - rows.back().width()) {
+                       Row & rb = rows.back();
+                       rb.push_back(*fcit);
+                       // if the row is too large, try to cut at last 
separator. In case
+                       // of success, reset indication that the row was broken 
abruptly.
+                       int const next_width = max_width_ - 
leftMargin(rb.pit(), rb.endpos())
+                               - rightMargin(rb.pit());
+
+                       Row::Elements next_elts = rb.shortenIfNeeded(width, 
next_width);
+
+                       // Go to next element
+                       ++fcit;
+
+                       // Handle later the elements returned by 
shortenIfNeeded.
+                       if (!next_elts.empty()) {
+                               rb.flushed(false);
+                               fcit.put(next_elts);
+                               need_new_row = true;
+                       }
+               } else {
+                       // a new element in the row
+                       rows.back().push_back(elt);
+                       rows.back().finalizeLast();
+                       pos = elt.endpos;
+
+                       // Go to next element
+                       ++fcit;
+
+                       // Add a new next element on the pile
+                       if (next_elt.isValid()) {
+                               // do as if we inserted this element in the 
original row
+                               fcit.put(next_elt);
+                               need_new_row = true;
+                       }
                }
        }
 
@@ -1146,185 +1172,6 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
        return rows;
 }
 
-/** This is the function where the hard work is done. The code here is
- * very sensitive to small changes :) Note that part of the
- * intelligence is also in Row::shortenIfNeeded.
- */
-bool TextMetrics::breakRow(Row & row, int const right_margin) const
-{
-       LATTEST(row.empty());//
-       Paragraph const & par = text_->getPar(row.pit());//
-       Buffer const & buf = text_->inset().buffer();//
-       BookmarksSection::BookmarkPosList bpl =//
-               theSession().bookmarks().bookmarksInPar(buf.fileName(), 
par.id());//
-
-       pos_type const end = par.size();//
-       pos_type const pos = row.pos();//
-       pos_type const body_pos = par.beginOfBody();//
-       bool const is_rtl = text_->isRTL(row.pit());//
-       bool need_new_row = false;//
-
-       row.left_margin = leftMargin(row.pit(), pos);//
-       row.right_margin = right_margin;//
-       if (is_rtl)//
-               swap(row.left_margin, row.right_margin);//
-       // Remember that the row width takes into account the left_margin
-       // but not the right_margin.
-       row.dim().wid = row.left_margin;//
-       // the width available for the row.
-       int const width = max_width_ - row.right_margin;//
-
-       // check for possible inline completion
-       DocIterator const & ic_it = bv_->inlineCompletionPos();//
-       pos_type ic_pos = -1;//
-       if (ic_it.inTexted() && ic_it.text() == text_ && ic_it.pit() == 
row.pit())//
-               ic_pos = ic_it.pos();//
-
-       // Now we iterate through until we reach the right margin
-       // or the end of the par, then build a representation of the row.
-       pos_type i = 
pos;//---------------------------------------------------vvv
-       FontIterator fi = FontIterator(*this, par, row.pit(), pos);
-       // The real stopping condition is a few lines below.
-       while (true) {
-               // Firstly, check whether there is a bookmark here.
-               if (lyxrc.bookmarks_visibility == LyXRC::BMK_INLINE)
-                       for (auto const & bp_p : bpl)
-                               if (bp_p.second == i) {
-                                       Font f = *fi;
-                                       f.fontInfo().setColor(Color_bookmark);
-                                       // ❶ U+2776 DINGBAT NEGATIVE CIRCLED 
DIGIT ONE
-                                       char_type const ch = 0x2775 + 
bp_p.first;
-                                       row.addVirtual(i, docstring(1, ch), f, 
Change());
-                               }
-
-               // The stopping condition is here so that the display of a
-               // bookmark can take place at paragraph start too.
-               if (i >= end || (i != pos && row.width() > width))//^width
-                       break;
-
-               char_type c = par.getChar(i);
-               // The most special cases are handled first.
-               if (par.isInset(i)) {
-                       Inset const * ins = par.getInset(i);
-                       Dimension dim = bv_->coordCache().insets().dim(ins);
-                       row.add(i, ins, dim, *fi, par.lookupChange(i));
-               } else if (c == ' ' && i + 1 == body_pos) {
-                       // There is a space at i, but it should not be
-                       // added as a separator, because it is just
-                       // before body_pos. Instead, insert some spacing to
-                       // align text
-                       FontMetrics const & fm = 
theFontMetrics(text_->labelFont(par));
-                       // this is needed to make sure that the row width is 
correct
-                       row.finalizeLast();
-                       int const add = max(fm.width(par.layout().labelsep),
-                                           labelEnd(row.pit()) - row.width());
-                       row.addSpace(i, add, *fi, par.lookupChange(i));
-               } else if (c == '\t')
-                       row.addSpace(i, theFontMetrics(*fi).width(from_ascii("  
  ")),
-                                    *fi, par.lookupChange(i));
-               else if (c == 0x2028 || c == 0x2029) {
-                       /**
-                        * U+2028 LINE SEPARATOR
-                        * U+2029 PARAGRAPH SEPARATOR
-
-                        * These are special unicode characters that break
-                        * lines/pragraphs. Not handling them lead to trouble 
wrt
-                        * Qt QTextLayout formatting. We add a visible character
-                        * on screen so that the user can see that something is
-                        * happening.
-                       */
-                       row.finalizeLast();
-                       // ⤶ U+2936 ARROW POINTING DOWNWARDS THEN CURVING 
LEFTWARDS
-                       // ¶ U+00B6 PILCROW SIGN
-                       char_type const screen_char = (c == 0x2028) ? 0x2936 : 
0x00B6;
-                       row.add(i, screen_char, *fi, par.lookupChange(i), i >= 
body_pos);
-               } else
-                       row.add(i, c, *fi, par.lookupChange(i), i >= body_pos);
-
-               // add inline completion width
-               // draw logically behind the previous character
-               if (ic_pos == i + 1 && !bv_->inlineCompletion().empty()) {
-                       docstring const comp = bv_->inlineCompletion();
-                       size_t const uniqueTo 
=bv_->inlineCompletionUniqueChars();
-                       Font f = *fi;
-
-                       if (uniqueTo > 0) {
-                               f.fontInfo().setColor(Color_inlinecompletion);
-                               row.addVirtual(i + 1, comp.substr(0, uniqueTo), 
f, Change());
-                       }
-                       f.fontInfo().setColor(Color_nonunique_inlinecompletion);
-                       row.addVirtual(i + 1, comp.substr(uniqueTo), f, 
Change());
-               }
-
-               // Handle some situations that abruptly terminate the row
-               // - Before an inset with BreakBefore
-               // - After an inset with BreakAfter
-               Inset const * prevInset = !row.empty() ? row.back().inset : 0;
-               Inset const * nextInset = (i + 1 < end) ? par.getInset(i + 1) : 
0;
-               if ((nextInset && nextInset->rowFlags() & BreakBefore)
-                   || (prevInset && prevInset->rowFlags() & BreakAfter)) {
-                       row.flushed(true);
-                       // Force a row creation after this one if it is ended by
-                       // an inset that either
-                       // - has row flag RowAfter that enforces that;
-                       // - or (1) did force the row breaking, (2) is at end of
-                       //   paragraph and (3) the said paragraph has an end 
label.
-                       need_new_row = prevInset &&
-                               (prevInset->rowFlags() & AlwaysBreakAfter
-                                || (prevInset->rowFlags() & BreakAfter && i + 
1 == end
-                                    && text_->getEndLabel(row.pit()) != 
END_LABEL_NO_LABEL));
-                       ++i;
-                       break;
-               }
-
-               ++i;
-               ++fi;
-       }
-       row.finalizeLast();
-       row.endpos(i);
-
-       // End of paragraph marker. The logic here is almost the
-       // same as in redoParagraph, remember keep them in sync.
-       ParagraphList const & pars = text_->paragraphs();
-       Change const & change = par.lookupChange(i);
-       if ((lyxrc.paragraph_markers || change.changed())
-           && !need_new_row // not this
-           && i == end && size_type(row.pit() + 1) < pars.size()) {
-               // add a virtual element for the end-of-paragraph
-               // marker; it is shown on screen, but does not exist
-               // in the paragraph.
-               Font f(text_->layoutFont(row.pit()));
-               f.fontInfo().setColor(Color_paragraphmarker);
-               f.setLanguage(par.getParLanguage(buf.params()));
-               // ¶ U+00B6 PILCROW SIGN
-               row.addVirtual(end, docstring(1, char_type(0x00B6)), f, change);
-       }
-
-       // Is there a end-of-paragaph change?
-       if (i == end && par.lookupChange(end).changed() && !need_new_row)
-               row.needsChangeBar(true);
-    //--------------------------------------------------------------------^^^
-       // FIXME : nothing below this
-
-       // if the row is too large, try to cut at last separator. In case
-       // of success, reset indication that the row was broken abruptly.
-       int const next_width = max_width_ - leftMargin(row.pit(), row.endpos())
-               - rightMargin(row.pit());
-
-       if (row.shortenIfNeeded(width, next_width))
-               row.flushed(false);
-       row.right_boundary(!row.empty() && row.endpos() < end//
-                          && row.back().endpos == row.endpos());//
-       // Last row in paragraph is flushed
-       if (row.endpos() == end)//
-               row.flushed(true);//
-
-       // make sure that the RTL elements are in reverse ordering
-       row.reverseRTL(is_rtl);//
-       //LYXERR0("breakrow: row is " << row);
-
-       return need_new_row;
-}
 
 int TextMetrics::parTopSpacing(pit_type const pit) const
 {
diff --git a/src/TextMetrics.h b/src/TextMetrics.h
index 3ff1161..e38ba71 100644
--- a/src/TextMetrics.h
+++ b/src/TextMetrics.h
@@ -151,15 +151,12 @@ private:
        /// FIXME??
        int labelEnd(pit_type const pit) const;
 
+       // Turn paragraph oh index \c pit into a single row
        Row tokenizeParagraph(pit_type pit) const;
 
+       // Break the row produced by tokenizeParagraph() into a list of rows.
        RowList breakParagraph(Row const & row) const;
 
-       /// sets row.end to the pos value *after* which a row should break.
-       /// for example, the pos after which isNewLine(pos) == true
-       /// \return true when another row is required (after a newline)
-       bool breakRow(Row & row, int right_margin) const;
-
        // Expands the alignment of row \param row in paragraph \param par
        LyXAlignment getAlign(Paragraph const & par, Row const & row) const;
        /// Aligns properly the row contents (computes spaces and fills)

commit bf1b00ae103edab3bc11d057d5a8bd946aa0befe
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Sun Jul 18 01:09:33 2021 +0200

    Implement handling of row_flags for row breaking
    
    To this end, add the helper function needsRowBreak which computes the
    effect of two consecutive row flags. This function implements the
    priorities described in RowFlags.h.
    
    This function is called with the relevant flags, or NoBreak* when at
    boundaries and updates need_new_row.
    
    Some common code is factored in a new cleanupRow() helper.

diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index fa855e4..1a6927a 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -980,22 +980,6 @@ Row TextMetrics::tokenizeParagraph(pit_type const pit) 
const
 
 namespace {
 
-Row newRow(TextMetrics const & tm, pit_type pit, pos_type pos, bool is_rtl)
-{
-       Row nrow;
-       nrow.pit(pit);
-       nrow.pos(pos);
-       nrow.left_margin = tm.leftMargin(pit, pos);
-       nrow.right_margin = tm.rightMargin(pit);
-       if (is_rtl)
-               swap(nrow.left_margin, nrow.right_margin);
-       // Remember that the row width takes into account the left_margin
-       // but not the right_margin.
-       nrow.dim().wid = nrow.left_margin;
-       return nrow;
-}
-
-
 /** Helper template flexible_const_iterator<T>
  * A way to iterate over a const container, but insert fake elements in it.
  * In the case of a row, we will have to break some elements, which
@@ -1018,6 +1002,8 @@ public:
 
        value_type operator*() const { return pile_.empty() ? *cit_ : 
pile_.back(); }
 
+       value_type const * operator->() const { return pile_.empty() ? &*cit_ : 
&pile_.back(); }
+
        void put(value_type const & e) { pile_.push_back(e); }
 
 // This should be private, but declaring the friend functions is too much work
@@ -1051,6 +1037,44 @@ bool operator==(flexible_const_iterator<T> const & t1,
        return t1.cit_ == t2.cit_ && t1.pile_.empty() && t2.pile_.empty();
 }
 
+Row newRow(TextMetrics const & tm, pit_type pit, pos_type pos, bool is_rtl)
+{
+       Row nrow;
+       nrow.pit(pit);
+       nrow.pos(pos);
+       nrow.left_margin = tm.leftMargin(pit, pos);
+       nrow.right_margin = tm.rightMargin(pit);
+       if (is_rtl)
+               swap(nrow.left_margin, nrow.right_margin);
+       // Remember that the row width takes into account the left_margin
+       // but not the right_margin.
+       nrow.dim().wid = nrow.left_margin;
+       return nrow;
+}
+
+
+void cleanupRow(Row & row, pos_type pos, pos_type real_endpos, bool is_rtl)
+{
+       row.endpos(pos);
+       row.right_boundary(!row.empty() && pos < real_endpos
+                          && row.back().endpos == pos);
+       // make sure that the RTL elements are in reverse ordering
+       row.reverseRTL(is_rtl);
+}
+
+// Implement the priorities described in RowFlags.h.
+bool needsRowBreak(int f1, int f2)
+{
+       if (f1 & AlwaysBreakAfter /*|| f2 & AlwaysBreakBefore*/)
+               return true;
+       if (f1 & NoBreakAfter || f2 & NoBreakBefore)
+               return false;
+       if (f1 & BreakAfter || f2 & BreakBefore)
+               return true;
+       return false;
+}
+
+
 }
 
 
@@ -1058,6 +1082,7 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
 {
        RowList rows;
        bool const is_rtl = text_->isRTL(bigrow.pit());
+       bool const end_label = text_->getEndLabel(bigrow.pit()) != 
END_LABEL_NO_LABEL;
 
        bool need_new_row = true;
        pos_type pos = 0;
@@ -1065,15 +1090,21 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
        flexible_const_iterator<Row> fcit = flexible_begin(bigrow);
        flexible_const_iterator<Row> const end = flexible_end(bigrow);
        while (true) {
+               bool const has_row = !rows.empty();
+               bool const row_empty = !has_row || rows.back().empty();
+               // The row flags of previous element, if there is one.
+               // Otherwise we use NoBreakAfter to avoid an empty row before
+               // e.g. a displayed equation.
+               int const f1 = row_empty ? NoBreakAfter : 
rows.back().back().row_flags;
+               // The row flags of next element, if there is one.
+               // Otherwise we use NoBreakBefore (see above), unless the
+               // paragraph has an end label (for which an empty row is OK).
+               int const f2 = (fcit == end) ? (end_label ? Inline : 
NoBreakBefore)
+                                            : fcit->row_flags;
+               need_new_row |= needsRowBreak(f1, f2);
                if (need_new_row) {
-                       if (!rows.empty()) {
-                               Row & rb = rows.back();
-                               rb.endpos(pos);
-                               rb.right_boundary(!rb.empty() && rb.endpos() < 
bigrow.endpos()
-                                                                  && 
rb.back().endpos == rb.endpos());
-                               // make sure that the RTL elements are in 
reverse ordering
-                               rb.reverseRTL(is_rtl);
-                       }
+                       if (!rows.empty())
+                               cleanupRow(rows.back(), pos, bigrow.endpos(), 
is_rtl);
                        rows.push_back(newRow(*this, bigrow.pit(), pos, 
is_rtl));
                        // the width available for the row.
                        width = max_width_ - rows.back().right_margin;
@@ -1107,13 +1138,9 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
        }
 
        if (!rows.empty()) {
-               Row & rb = rows.back();
+               cleanupRow(rows.back(), pos, bigrow.endpos(), is_rtl);
                // Last row in paragraph is flushed
-               rb.flushed(true);
-               rb.endpos(bigrow.endpos());
-               rb.right_boundary(false);
-               // make sure that the RTL elements are in reverse ordering
-               rb.reverseRTL(is_rtl);
+               rows.back().flushed(true);
        }
 
        return rows;
@@ -1227,9 +1254,8 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
                        }
                        f.fontInfo().setColor(Color_nonunique_inlinecompletion);
                        row.addVirtual(i + 1, comp.substr(uniqueTo), f, 
Change());
-               
}//---------------------------------------------------------------^^^
+               }
 
-               // FIXME: Handle when breaking the rows
                // Handle some situations that abruptly terminate the row
                // - Before an inset with BreakBefore
                // - After an inset with BreakAfter
@@ -1254,7 +1280,6 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
                ++i;
                ++fi;
        }
-       
//--------------------------------------------------------------------vvv
        row.finalizeLast();
        row.endpos(i);
 

commit 4a32327af993aeb1f436d78fc5d2728a1685c759
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Sat Jul 17 23:16:15 2021 +0200

    Change the way the element's width is updated.
    
    Remove the code that computed the width every 30 characters (yay!).
    Make sure that finalizeLast() is called after inserting a row element in
    a row in breakParagraph.

diff --git a/src/Row.cpp b/src/Row.cpp
index 705a147..3bc8ef0 100644
--- a/src/Row.cpp
+++ b/src/Row.cpp
@@ -370,8 +370,7 @@ void Row::finalizeLast()
        if (elt.change.changed())
                changebar_ = true;
 
-       if (elt.type == STRING) {
-               dim_.wid -= elt.dim.wid;
+       if (elt.type == STRING && elt.dim.wid == 0) {
                elt.dim.wid = theFontMetrics(elt.font).width(elt.str);
                dim_.wid += elt.dim.wid;
        }
@@ -401,16 +400,8 @@ void Row::add(pos_type const pos, char_type const c,
                e.row_flags = can_break ? CanBreakInside : Inline;
                elements_.push_back(e);
        }
-       if (back().str.length() % 30 == 0) {
-               dim_.wid -= back().dim.wid;
-               back().str += c;
-               back().endpos = pos + 1;
-               back().dim.wid = theFontMetrics(back().font).width(back().str);
-               dim_.wid += back().dim.wid;
-       } else {
-               back().str += c;
-               back().endpos = pos + 1;
-       }
+       back().str += c;
+       back().endpos = pos + 1;
 }
 
 
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index b5ba974..fa855e4 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -1092,6 +1092,7 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
                                                    
!elt.font.language()->wordWrap());
                // a new element in the row
                rows.back().push_back(elt);
+               rows.back().finalizeLast();
                pos = elt.endpos;
 
                // Go to next element

commit bcddde16c90cd32d2d1695179a5e9e1f0c101b71
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Sat Jul 17 02:31:49 2021 +0200

    Introduce helper template to simplify breakParagraph code
    
    This is a semi-generic iterator for iterating over a container and
    pretend that we add elements to it along the way.

diff --git a/src/Row.h b/src/Row.h
index 4fdcee4..72d7a86 100644
--- a/src/Row.h
+++ b/src/Row.h
@@ -150,6 +150,8 @@ public:
                friend std::ostream & operator<<(std::ostream & os, Element 
const & row);
        };
 
+       ///
+       typedef Element value_type;
 
        ///
        Row() {}
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index 00d65da..b5ba974 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -995,6 +995,62 @@ Row newRow(TextMetrics const & tm, pit_type pit, pos_type 
pos, bool is_rtl)
        return nrow;
 }
 
+
+/** Helper template flexible_const_iterator<T>
+ * A way to iterate over a const container, but insert fake elements in it.
+ * In the case of a row, we will have to break some elements, which
+ * create new ones. This class allows to abstract this.
+ * Only the required parts are implemented for now.
+ */
+template<class T>
+class flexible_const_iterator {
+       typedef typename T::value_type value_type;
+public:
+
+       //
+       flexible_const_iterator operator++() {
+               if (pile_.empty())
+                       ++cit_;
+               else
+                       pile_.pop_back();
+               return *this;
+       }
+
+       value_type operator*() const { return pile_.empty() ? *cit_ : 
pile_.back(); }
+
+       void put(value_type const & e) { pile_.push_back(e); }
+
+// This should be private, but declaring the friend functions is too much work
+//private:
+       typename T::const_iterator cit_;
+       // A vector that is used as like a pile to store the elements to
+       // consider before incrementing the underlying iterator.
+       vector<value_type> pile_;
+};
+
+
+template<class T>
+flexible_const_iterator<T> flexible_begin(T const & t)
+{
+       return { t.begin(), vector<typename T::value_type>() };
+}
+
+
+template<class T>
+flexible_const_iterator<T> flexible_end(T const & t)
+{
+       return { t.end(), vector<typename T::value_type>() };
+}
+
+
+// Equality is only possible if respective piles are empty
+template<class T>
+bool operator==(flexible_const_iterator<T> const & t1,
+                flexible_const_iterator<T> const & t2)
+{
+       return t1.cit_ == t2.cit_ && t1.pile_.empty() && t2.pile_.empty();
+}
+
 }
 
 
@@ -1006,11 +1062,8 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
        bool need_new_row = true;
        pos_type pos = 0;
        int width = 0;
-       Row::const_iterator cit = bigrow.begin();
-       Row::const_iterator const end = bigrow.end();
-       // This is a vector, but we use it like a pile putting and taking
-       // stuff at the back.
-       Row::Elements pile;
+       flexible_const_iterator<Row> fcit = flexible_begin(bigrow);
+       flexible_const_iterator<Row> const end = flexible_end(bigrow);
        while (true) {
                if (need_new_row) {
                        if (!rows.empty()) {
@@ -1029,27 +1082,25 @@ RowList TextMetrics::breakParagraph(Row const & bigrow) 
const
 
                // The stopping condition is here because we may need a new
                // empty row at the end.
-               if (cit == end && pile.empty())
+               if (fcit == end)
                        break;
 
                // Next element to consider is either the top of the temporary
                // pile, or the place when we were in main row
-               Row::Element elt = pile.empty() ? *cit : pile.back();
-               //LYXERR0("elt=" << elt);
+               Row::Element elt = *fcit;
                Row::Element next_elt = elt.splitAt(width - rows.back().width(),
                                                    
!elt.font.language()->wordWrap());
-               //LYXERR0("next_elt=" << next_elt);
                // a new element in the row
                rows.back().push_back(elt);
                pos = elt.endpos;
+
                // Go to next element
-               if (pile.empty())
-                       ++cit;
-               else
-                       pile.pop_back();
+               ++fcit;
+
                // Add a new next element on the pile
                if (next_elt.isValid()) {
-                       pile.push_back(next_elt);
+                       // do as if we inserted this element in the original row
+                       fcit.put(next_elt);
                        need_new_row = true;
                }
        }

commit 639dc69eea04fb3448030b815fa402deaad1527d
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Fri Jul 16 00:10:25 2021 +0200

    A set of easy fixes and missing features
    
    * show changebar when end of paragraph is changed.
    
    * when row is finished, set endpos and right_boundary
    
    * handle bidi.

diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index 1d36d7a..00d65da 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -955,12 +955,15 @@ Row TextMetrics::tokenizeParagraph(pit_type const pit) 
const
        row.finalizeLast();
        row.endpos(end);
 
-       // End of paragraph marker. The logic here is almost the
+       // End of paragraph marker, either if LyXRc requires it, or there
+       // is an end of paragraph change. The logic here is almost the
        // same as in redoParagraph, remember keep them in sync.
        ParagraphList const & pars = text_->paragraphs();
-       Change const & change = par.lookupChange(i);
-       if ((lyxrc.paragraph_markers || change.changed())
-           && i == end && size_type(pit + 1) < pars.size()) {
+       Change const & endchange = par.lookupChange(end);
+       if (endchange.changed())
+               row.needsChangeBar(true);
+       if ((lyxrc.paragraph_markers || endchange.changed())
+           && size_type(pit + 1) < pars.size()) {
                // add a virtual element for the end-of-paragraph
                // marker; it is shown on screen, but does not exist
                // in the paragraph.
@@ -968,7 +971,7 @@ Row TextMetrics::tokenizeParagraph(pit_type const pit) const
                f.fontInfo().setColor(Color_paragraphmarker);
                f.setLanguage(par.getParLanguage(buf.params()));
                // ¶ U+00B6 PILCROW SIGN
-               row.addVirtual(end, docstring(1, char_type(0x00B6)), f, change);
+               row.addVirtual(end, docstring(1, char_type(0x00B6)), f, 
endchange);
        }
 
        return row;
@@ -995,24 +998,30 @@ Row newRow(TextMetrics const & tm, pit_type pit, pos_type 
pos, bool is_rtl)
 }
 
 
-RowList TextMetrics::breakParagraph(Row const & row) const
+RowList TextMetrics::breakParagraph(Row const & bigrow) const
 {
        RowList rows;
-       bool const is_rtl = text_->isRTL(row.pit());
+       bool const is_rtl = text_->isRTL(bigrow.pit());
 
        bool need_new_row = true;
        pos_type pos = 0;
        int width = 0;
-       Row::const_iterator cit = row.begin();
-       Row::const_iterator const end = row.end();
+       Row::const_iterator cit = bigrow.begin();
+       Row::const_iterator const end = bigrow.end();
        // This is a vector, but we use it like a pile putting and taking
        // stuff at the back.
        Row::Elements pile;
        while (true) {
                if (need_new_row) {
-                       if (!rows.empty())
-                               rows.back().endpos(pos);
-                       rows.push_back(newRow(*this, row.pit(), pos, is_rtl));
+                       if (!rows.empty()) {
+                               Row & rb = rows.back();
+                               rb.endpos(pos);
+                               rb.right_boundary(!rb.empty() && rb.endpos() < 
bigrow.endpos()
+                                                                  && 
rb.back().endpos == rb.endpos());
+                               // make sure that the RTL elements are in 
reverse ordering
+                               rb.reverseRTL(is_rtl);
+                       }
+                       rows.push_back(newRow(*this, bigrow.pit(), pos, 
is_rtl));
                        // the width available for the row.
                        width = max_width_ - rows.back().right_margin;
                        need_new_row = false;
@@ -1045,6 +1054,16 @@ RowList TextMetrics::breakParagraph(Row const & row) 
const
                }
        }
 
+       if (!rows.empty()) {
+               Row & rb = rows.back();
+               // Last row in paragraph is flushed
+               rb.flushed(true);
+               rb.endpos(bigrow.endpos());
+               rb.right_boundary(false);
+               // make sure that the RTL elements are in reverse ordering
+               rb.reverseRTL(is_rtl);
+       }
+
        return rows;
 }
 
@@ -1217,14 +1236,14 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
 
        if (row.shortenIfNeeded(width, next_width))
                row.flushed(false);
-       row.right_boundary(!row.empty() && row.endpos() < end
-                          && row.back().endpos == row.endpos());
+       row.right_boundary(!row.empty() && row.endpos() < end//
+                          && row.back().endpos == row.endpos());//
        // Last row in paragraph is flushed
-       if (row.endpos() == end)
-               row.flushed(true);
+       if (row.endpos() == end)//
+               row.flushed(true);//
 
        // make sure that the RTL elements are in reverse ordering
-       row.reverseRTL(is_rtl);
+       row.reverseRTL(is_rtl);//
        //LYXERR0("breakrow: row is " << row);
 
        return need_new_row;

commit 25a75e5d6742e4dc024f9a975ed85ef2e251bbce
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Wed Jul 14 00:48:03 2021 +0200

    Use the new tokenizing and breaking code instead of breakRow.

diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index fb6574d..1d36d7a 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -516,43 +516,28 @@ bool TextMetrics::redoParagraph(pit_type const pit, bool 
const align_rows)
                }
        }
 
-       pos_type first = 0;
-       size_t row_index = 0;
-       bool need_new_row = false;
-       // maximum pixel width of a row
-       do {
-               if (row_index == pm.rows().size())
-                       pm.rows().push_back(Row());
-               else
-                       pm.rows()[row_index] = Row();
-               Row & row = pm.rows()[row_index];
-               row.pit(pit);
-               row.pos(first);
-               need_new_row = breakRow(row, right_margin);
+       // Transform the paragraph into a single row containing all the 
elements.
+       Row const bigrow = tokenizeParagraph(pit);
+       // Split the row in several rows fitting in available width
+       pm.rows() = breakParagraph(bigrow);
+
+       /* If there is more than one row, expand the text to the full
+        * allowable width. This setting here is needed for the
+        * setRowAlignment() below. We do nothing when tight insets are
+        * requested.
+        */
+       if (pm.rows().size() > 1 && !tight_ && dim_.wid < max_width_)
+                       dim_.wid = max_width_;
+
+       // Compute height and alignment of the rows.
+       for (Row & row : pm.rows()) {
                setRowHeight(row);
-               row.changed(true);
-               if ((row_index || row.endpos() < par.size() || 
row.right_boundary())
-                   && !tight_) {
-                       /* If there is more than one row or the row has been
-                        * broken by a display inset or a newline, expand the 
text
-                        * to the full allowable width. This setting here is
-                        * needed for the setRowAlignment() below.
-                        * We do nothing when tight insets are requested.
-                        */
-                       if (dim_.wid < max_width_)
-                               dim_.wid = max_width_;
-               }
                if (align_rows)
                        setRowAlignment(row, max(dim_.wid, row.width()));
-               first = row.endpos();
-               ++row_index;
 
                pm.dim().wid = max(pm.dim().wid, row.width() + 
row.right_margin);
                pm.dim().des += row.height();
-       } while (first < par.size() || need_new_row);
-
-       if (row_index < pm.rows().size())
-               pm.rows().resize(row_index);
+       }
 
        // This type of margin can only be handled at the global paragraph level
        if (par.layout().margintype == MARGIN_RIGHT_ADDRESS_BOX) {

commit 0a1bbb23bb65d8fa59a8f244fe8c520f89571265
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Wed Jul 14 00:47:42 2021 +0200

    Break the paragraph's big row according to margins
    
    Still many features missing:
    - handle insets that break rows (display math, newline, ...)
    - handle rows that are too long by replacing the single call to
      breakAt with a call to a reworked Row::shortenIfNeeded.
    - some easy things at the end of breakRow (bidi text, etc.).

diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index cc380cc..fb6574d 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -990,6 +990,79 @@ Row TextMetrics::tokenizeParagraph(pit_type const pit) 
const
 }
 
 
+namespace {
+
+Row newRow(TextMetrics const & tm, pit_type pit, pos_type pos, bool is_rtl)
+{
+       Row nrow;
+       nrow.pit(pit);
+       nrow.pos(pos);
+       nrow.left_margin = tm.leftMargin(pit, pos);
+       nrow.right_margin = tm.rightMargin(pit);
+       if (is_rtl)
+               swap(nrow.left_margin, nrow.right_margin);
+       // Remember that the row width takes into account the left_margin
+       // but not the right_margin.
+       nrow.dim().wid = nrow.left_margin;
+       return nrow;
+}
+
+}
+
+
+RowList TextMetrics::breakParagraph(Row const & row) const
+{
+       RowList rows;
+       bool const is_rtl = text_->isRTL(row.pit());
+
+       bool need_new_row = true;
+       pos_type pos = 0;
+       int width = 0;
+       Row::const_iterator cit = row.begin();
+       Row::const_iterator const end = row.end();
+       // This is a vector, but we use it like a pile putting and taking
+       // stuff at the back.
+       Row::Elements pile;
+       while (true) {
+               if (need_new_row) {
+                       if (!rows.empty())
+                               rows.back().endpos(pos);
+                       rows.push_back(newRow(*this, row.pit(), pos, is_rtl));
+                       // the width available for the row.
+                       width = max_width_ - rows.back().right_margin;
+                       need_new_row = false;
+               }
+
+               // The stopping condition is here because we may need a new
+               // empty row at the end.
+               if (cit == end && pile.empty())
+                       break;
+
+               // Next element to consider is either the top of the temporary
+               // pile, or the place when we were in main row
+               Row::Element elt = pile.empty() ? *cit : pile.back();
+               //LYXERR0("elt=" << elt);
+               Row::Element next_elt = elt.splitAt(width - rows.back().width(),
+                                                   
!elt.font.language()->wordWrap());
+               //LYXERR0("next_elt=" << next_elt);
+               // a new element in the row
+               rows.back().push_back(elt);
+               pos = elt.endpos;
+               // Go to next element
+               if (pile.empty())
+                       ++cit;
+               else
+                       pile.pop_back();
+               // Add a new next element on the pile
+               if (next_elt.isValid()) {
+                       pile.push_back(next_elt);
+                       need_new_row = true;
+               }
+       }
+
+       return rows;
+}
+
 /** This is the function where the hard work is done. The code here is
  * very sensitive to small changes :) Note that part of the
  * intelligence is also in Row::shortenIfNeeded.
@@ -1003,20 +1076,20 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
                theSession().bookmarks().bookmarksInPar(buf.fileName(), 
par.id());//
 
        pos_type const end = par.size();//
-       pos_type const pos = row.pos();
+       pos_type const pos = row.pos();//
        pos_type const body_pos = par.beginOfBody();//
-       bool const is_rtl = text_->isRTL(row.pit());
-       bool need_new_row = false;
+       bool const is_rtl = text_->isRTL(row.pit());//
+       bool need_new_row = false;//
 
-       row.left_margin = leftMargin(row.pit(), pos);
-       row.right_margin = right_margin;
-       if (is_rtl)
-               swap(row.left_margin, row.right_margin);
+       row.left_margin = leftMargin(row.pit(), pos);//
+       row.right_margin = right_margin;//
+       if (is_rtl)//
+               swap(row.left_margin, row.right_margin);//
        // Remember that the row width takes into account the left_margin
        // but not the right_margin.
-       row.dim().wid = row.left_margin;
+       row.dim().wid = row.left_margin;//
        // the width available for the row.
-       int const width = max_width_ - row.right_margin;
+       int const width = max_width_ - row.right_margin;//
 
        // check for possible inline completion
        DocIterator const & ic_it = bv_->inlineCompletionPos();//
diff --git a/src/TextMetrics.h b/src/TextMetrics.h
index 1666cd0..3ff1161 100644
--- a/src/TextMetrics.h
+++ b/src/TextMetrics.h
@@ -153,6 +153,8 @@ private:
 
        Row tokenizeParagraph(pit_type pit) const;
 
+       RowList breakParagraph(Row const & row) const;
+
        /// sets row.end to the pos value *after* which a row should break.
        /// for example, the pos after which isNewLine(pos) == true
        /// \return true when another row is required (after a newline)

commit a8aad2905805ac2804b73fe950e9f37fb6ff6d8a
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Mon Jul 12 00:07:59 2021 +0200

    Implement Row::Element::row_flags
    
    Move the enum definition RowFlags in its own include file, to avoid
    loading Inset.h. Document it more thoroughly.
    
    Rename RowAfter to AlwaysBreakAfter.
    
    Add CanBreakInside (rows that can be themselves broken). This allow to
    differentiate elements before bodyPos() and allows to remove a
    parameter to shortenIfNeeded().
    
    Make the Inset::rowFlags() method return int instead of RowFlags, as
    should be done for all the bitwise flags. Remove the hand-made bitwise
    operators.
    
    Set R::E::row_flags when creating elements.
    * INSET elements use the inset's rowFLags();
    * virtual element forbid breaking before them, and inherit the *After
      flags from the previous element of the row;
    * STRING elements usr CanBreakInside, except before bodyPos.
    
    More stuff may be added later.

diff --git a/src/Makefile.am b/src/Makefile.am
index 31701f2..99a0f98 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -273,6 +273,7 @@ HEADERFILESCORE = \
        ParIterator.h \
        PDFOptions.h \
        Row.h \
+       RowFlags.h \
        RowPainter.h \
        Server.h \
        ServerSocket.h \
diff --git a/src/Row.cpp b/src/Row.cpp
index da2d885..705a147 100644
--- a/src/Row.cpp
+++ b/src/Row.cpp
@@ -135,7 +135,7 @@ pos_type Row::Element::x2pos(int &x) const
 
 Row::Element Row::Element::splitAt(int w, bool force)
 {
-       if (type != STRING)
+       if (type != STRING || !(row_flags & CanBreakInside))
                return Element();
 
        FontMetrics const & fm = theFontMetrics(font);
@@ -145,6 +145,7 @@ Row::Element Row::Element::splitAt(int w, bool force)
                Element ret(STRING, pos + i, font, change);
                ret.str = str.substr(i);
                ret.endpos = ret.pos + ret.str.length();
+               ret.row_flags = row_flags & (CanBreakInside | AfterFlags);
                str.erase(i);
                endpos = pos + i;
                //lyxerr << "breakAt(" << w << ")  Row element Broken at " << x 
<< "(w(str)=" << fm.width(str) << "): e=" << *this << endl;
@@ -378,12 +379,13 @@ void Row::finalizeLast()
 
 
 void Row::add(pos_type const pos, Inset const * ins, Dimension const & dim,
-             Font const & f, Change const & ch)
+              Font const & f, Change const & ch)
 {
        finalizeLast();
        Element e(INSET, pos, f, ch);
        e.inset = ins;
        e.dim = dim;
+       e.row_flags = ins->rowFlags();
        elements_.push_back(e);
        dim_.wid += dim.wid;
        changebar_ |= ins->isChanged();
@@ -391,11 +393,12 @@ void Row::add(pos_type const pos, Inset const * ins, 
Dimension const & dim,
 
 
 void Row::add(pos_type const pos, char_type const c,
-             Font const & f, Change const & ch)
+              Font const & f, Change const & ch, bool can_break)
 {
        if (!sameString(f, ch)) {
                finalizeLast();
                Element e(STRING, pos, f, ch);
+               e.row_flags = can_break ? CanBreakInside : Inline;
                elements_.push_back(e);
        }
        if (back().str.length() % 30 == 0) {
@@ -420,6 +423,10 @@ void Row::addVirtual(pos_type const pos, docstring const & 
s,
        e.dim.wid = theFontMetrics(f).width(s);
        dim_.wid += e.dim.wid;
        e.endpos = pos;
+       // Copy after* flags from previous elements, forbid break before element
+       int const prev_row_flags = elements_.empty() ? Inline : 
elements_.back().row_flags;
+       int const can_inherit = AfterFlags & ~AlwaysBreakAfter;
+       e.row_flags = (prev_row_flags & can_inherit) | NoBreakBefore;
        elements_.push_back(e);
        finalizeLast();
 }
@@ -450,7 +457,7 @@ void Row::pop_back()
 }
 
 
-bool Row::shortenIfNeeded(pos_type const keep, int const w, int const 
next_width)
+bool Row::shortenIfNeeded(int const w, int const next_width)
 {
        if (empty() || width() <= w)
                return false;
@@ -482,11 +489,10 @@ bool Row::shortenIfNeeded(pos_type const keep, int const 
w, int const next_width
                // make a copy of the element to work on it.
                Element brk = *cit_brk;
                /* If the current element is an inset that allows breaking row
-                * after itself, and it the row is already short enough after
+                * after itself, and if the row is already short enough after
                 * this inset, then cut right after this element.
                 */
-               if (wid_brk <= w && brk.type == INSET
-                   && brk.inset->rowFlags() & Inset::CanBreakAfter) {
+               if (wid_brk <= w && brk.row_flags & CanBreakAfter) {
                        end_ = brk.endpos;
                        dim_.wid = wid_brk;
                        elements_.erase(cit_brk + 1, end);
@@ -504,10 +510,6 @@ bool Row::shortenIfNeeded(pos_type const keep, int const 
w, int const next_width
                 * not allowed at the beginning or end of line.
                */
                bool const word_wrap = brk.font.language()->wordWrap();
-               // When there is text before the body part (think description
-               // environment), do not try to break.
-               if (brk.pos < keep)
-                       continue;
                /* We have found a suitable separable element. This is the 
common case.
                 * Try to break it cleanly (at word boundary) at a length that 
is both
                 * - less than the available space on the row
diff --git a/src/Row.h b/src/Row.h
index 3048cf1..4fdcee4 100644
--- a/src/Row.h
+++ b/src/Row.h
@@ -18,6 +18,7 @@
 #include "Changes.h"
 #include "Dimension.h"
 #include "Font.h"
+#include "RowFlags.h"
 
 #include "support/docstring.h"
 #include "support/types.h"
@@ -143,6 +144,8 @@ public:
                Change change;
                // is it possible to add contents to this element?
                bool final = false;
+               // properties with respect to row breaking (made of RowFlag 
enums)
+               int row_flags = Inline;
 
                friend std::ostream & operator<<(std::ostream & os, Element 
const & row);
        };
@@ -247,7 +250,7 @@ public:
                 Font const & f, Change const & ch);
        ///
        void add(pos_type pos, char_type const c,
-                Font const & f, Change const & ch);
+                Font const & f, Change const & ch, bool can_break);
        ///
        void addVirtual(pos_type pos, docstring const & s,
                        Font const & f, Change const & ch);
@@ -287,12 +290,11 @@ public:
         * if row width is too large, remove all elements after last
         * separator and update endpos if necessary. If all that
         * remains is a large word, cut it to \param width.
-        * \param body_pos minimum amount of text to keep.
         * \param width maximum width of the row.
         * \param available width on next row.
         * \return true if the row has been shortened.
         */
-       bool shortenIfNeeded(pos_type const body_pos, int const width, int 
const next_width);
+       bool shortenIfNeeded(int const width, int const next_width);
 
        /**
         * If last element of the row is a string, compute its width
diff --git a/src/RowFlags.h b/src/RowFlags.h
new file mode 100644
index 0000000..f94f0c6
--- /dev/null
+++ b/src/RowFlags.h
@@ -0,0 +1,57 @@
+// -*- C++ -*-
+/**
+ * \file RowFlags.h
+ * This file is part of LyX, the document processor.
+ * Licence details can be found in the file COPYING.
+ *
+ * \author Jean-Marc Lasgouttes
+ *
+ * Full author contact details are available in file CREDITS.
+ */
+
+#ifndef ROWFLAGS_H
+#define ROWFLAGS_H
+
+// Do not include anything here
+
+namespace lyx {
+
+/* The list of possible flags, that can be combined.
+ * Some flags that should logically be here (e.g.,
+ * CanBreakBefore), do not exist. This is because the need has not
+ * been identitfied yet.
+ *
+ * Priorities when before/after disagree:
+ *      AlwaysBreak* > NoBreak* > Break* or CanBreak*.
+ */
+enum RowFlags {
+       // Do not break before or after this element, except if really
+       // needed (between NoBreak* and CanBreak*).
+       Inline = 0,
+       // break row before this element if the row is not empty
+       BreakBefore = 1 << 0,
+       // Avoid breaking row before this element
+       NoBreakBefore = 1 << 1,
+       // force new (maybe empty) row after this element
+       AlwaysBreakAfter = 1 << 2,
+       // break row after this element if there are more elements
+       BreakAfter = 1 << 3,
+       // break row whenever needed after this element
+       CanBreakAfter = 1 << 4,
+       // Avoid breaking row after this element
+       NoBreakAfter = 1 << 5,
+       // The contents of the row may be broken in two (e.g. string)
+       CanBreakInside = 1 << 6,
+       // specify an alignment (left, right) for a display element
+       // (default is center)
+       AlignLeft = 1 << 7,
+       AlignRight = 1 << 8,
+       // A display element breaks row at both ends
+       Display = BreakBefore | BreakAfter,
+       // Flags that concern breaking after element
+       AfterFlags = AlwaysBreakAfter | BreakAfter | CanBreakAfter | 
NoBreakAfter
+};
+
+} // namespace lyx
+
+#endif
diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index f14fe1f..cc380cc 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -634,10 +634,10 @@ LyXAlignment TextMetrics::getAlign(Paragraph const & par, 
Row const & row) const
 
        // Display-style insets should always be on a centered row
        if (Inset const * inset = par.getInset(row.pos())) {
-               if (inset->rowFlags() & Inset::Display) {
-                       if (inset->rowFlags() & Inset::AlignLeft)
+               if (inset->rowFlags() & Display) {
+                       if (inset->rowFlags() & AlignLeft)
                                align = LYX_ALIGN_BLOCK;
-                       else if (inset->rowFlags() & Inset::AlignRight)
+                       else if (inset->rowFlags() & AlignRight)
                                align = LYX_ALIGN_RIGHT;
                        else
                                align = LYX_ALIGN_CENTER;
@@ -928,7 +928,7 @@ Row TextMetrics::tokenizeParagraph(pit_type const pit) const
                        row.addSpace(i, add, *fi, par.lookupChange(i));
                } else if (c == '\t')
                        row.addSpace(i, theFontMetrics(*fi).width(from_ascii("  
  ")),
-                                    *fi, par.lookupChange(i));
+                                    *fi, par.lookupChange(i));
                else if (c == 0x2028 || c == 0x2029) {
                        /**
                         * U+2028 LINE SEPARATOR
@@ -944,9 +944,10 @@ Row TextMetrics::tokenizeParagraph(pit_type const pit) 
const
                        // ⤶ U+2936 ARROW POINTING DOWNWARDS THEN CURVING 
LEFTWARDS
                        // ¶ U+00B6 PILCROW SIGN
                        char_type const screen_char = (c == 0x2028) ? 0x2936 : 
0x00B6;
-                       row.add(i, screen_char, *fi, par.lookupChange(i));
+                       row.add(i, screen_char, *fi, par.lookupChange(i), i >= 
body_pos);
                } else
-                       row.add(i, c, *fi, par.lookupChange(i));
+                       // row elements before body are unbreakable
+                       row.add(i, c, *fi, par.lookupChange(i), i >= body_pos);
 
                // add inline completion width
                // draw logically behind the previous character
@@ -1080,9 +1081,9 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
                        // ⤶ U+2936 ARROW POINTING DOWNWARDS THEN CURVING 
LEFTWARDS
                        // ¶ U+00B6 PILCROW SIGN
                        char_type const screen_char = (c == 0x2028) ? 0x2936 : 
0x00B6;
-                       row.add(i, screen_char, *fi, par.lookupChange(i));
+                       row.add(i, screen_char, *fi, par.lookupChange(i), i >= 
body_pos);
                } else
-                       row.add(i, c, *fi, par.lookupChange(i));
+                       row.add(i, c, *fi, par.lookupChange(i), i >= body_pos);
 
                // add inline completion width
                // draw logically behind the previous character
@@ -1105,8 +1106,8 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
                // - After an inset with BreakAfter
                Inset const * prevInset = !row.empty() ? row.back().inset : 0;
                Inset const * nextInset = (i + 1 < end) ? par.getInset(i + 1) : 
0;
-               if ((nextInset && nextInset->rowFlags() & Inset::BreakBefore)
-                   || (prevInset && prevInset->rowFlags() & 
Inset::BreakAfter)) {
+               if ((nextInset && nextInset->rowFlags() & BreakBefore)
+                   || (prevInset && prevInset->rowFlags() & BreakAfter)) {
                        row.flushed(true);
                        // Force a row creation after this one if it is ended by
                        // an inset that either
@@ -1114,8 +1115,8 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
                        // - or (1) did force the row breaking, (2) is at end of
                        //   paragraph and (3) the said paragraph has an end 
label.
                        need_new_row = prevInset &&
-                               (prevInset->rowFlags() & Inset::RowAfter
-                                || (prevInset->rowFlags() & Inset::BreakAfter 
&& i + 1 == end
+                               (prevInset->rowFlags() & AlwaysBreakAfter
+                                || (prevInset->rowFlags() & BreakAfter && i + 
1 == end
                                     && text_->getEndLabel(row.pit()) != 
END_LABEL_NO_LABEL));
                        ++i;
                        break;
@@ -1156,7 +1157,7 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
        int const next_width = max_width_ - leftMargin(row.pit(), row.endpos())
                - rightMargin(row.pit());
 
-       if (row.shortenIfNeeded(body_pos, width, next_width))
+       if (row.shortenIfNeeded(width, next_width))
                row.flushed(false);
        row.right_boundary(!row.empty() && row.endpos() < end
                           && row.back().endpos == row.endpos());
@@ -1900,7 +1901,7 @@ int TextMetrics::leftMargin(pit_type const pit, pos_type 
const pos) const
            // display style insets do not need indentation
            && !(!par.empty()
                 && par.isInset(0)
-                && par.getInset(0)->rowFlags() & Inset::Display)
+                && par.getInset(0)->rowFlags() & Display)
            && (!(tclass.isDefaultLayout(par.layout())
                || tclass.isPlainLayout(par.layout()))
                || buffer.params().paragraph_separation
diff --git a/src/insets/Inset.h b/src/insets/Inset.h
index af24423..372b95a 100644
--- a/src/insets/Inset.h
+++ b/src/insets/Inset.h
@@ -20,6 +20,7 @@
 #include "LayoutEnums.h"
 #include "OutputEnums.h"
 #include "OutputParams.h"
+#include "RowFlags.h"
 
 #include "support/docstring.h"
 #include "support/strfwd.h"
@@ -478,26 +479,8 @@ public:
 
        virtual CtObject getCtObject(OutputParams const &) const;
 
-       enum RowFlags {
-               Inline = 0,
-               // break row before this inset
-               BreakBefore = 1 << 0,
-               // break row after this inset
-               BreakAfter = 1 << 1,
-               // it is possible to break after this inset
-               CanBreakAfter = 1 << 2,
-               // force new (maybe empty) row after this inset
-               RowAfter = 1 << 3,
-               // specify an alignment (left, right) for a display inset
-               // (default is center)
-               AlignLeft = 1 << 4,
-               AlignRight = 1 << 5,
-               // A display inset breaks row at both ends
-               Display = BreakBefore | BreakAfter
-       };
-
-       /// How should this inset be displayed in its row?
-       virtual RowFlags rowFlags() const { return Inline; }
+       // properties with respect to row breaking (made of RowFLag enums)
+       virtual int rowFlags() const { return Inline; }
        /// indentation before this inset (only needed for displayed hull 
insets with fleqn option)
        virtual int indent(BufferView const &) const { return 0; }
        ///
@@ -655,20 +638,6 @@ protected:
 };
 
 
-inline Inset::RowFlags operator|(Inset::RowFlags const d1,
-                                    Inset::RowFlags const d2)
-{
-       return static_cast<Inset::RowFlags>(int(d1) | int(d2));
-}
-
-
-inline Inset::RowFlags operator&(Inset::RowFlags const d1,
-                                    Inset::RowFlags const d2)
-{
-       return static_cast<Inset::RowFlags>(int(d1) & int(d2));
-}
-
-
 } // namespace lyx
 
 #endif
diff --git a/src/insets/InsetBibtex.h b/src/insets/InsetBibtex.h
index be7659f..55451f5 100644
--- a/src/insets/InsetBibtex.h
+++ b/src/insets/InsetBibtex.h
@@ -47,7 +47,7 @@ public:
        ///
        InsetCode lyxCode() const override { return BIBTEX_CODE; }
        ///
-       RowFlags rowFlags() const override { return Display; }
+       int rowFlags() const override { return Display; }
        ///
        void latex(otexstream &, OutputParams const &) const override;
        ///
diff --git a/src/insets/InsetCaption.h b/src/insets/InsetCaption.h
index ed6dbbb..c1bcd17 100644
--- a/src/insets/InsetCaption.h
+++ b/src/insets/InsetCaption.h
@@ -40,7 +40,7 @@ private:
        ///
        void write(std::ostream & os) const override;
        ///
-       RowFlags rowFlags() const override { return Display; }
+       int rowFlags() const override { return Display; }
        ///
        bool neverIndent() const override { return true; }
        ///
diff --git a/src/insets/InsetFloatList.h b/src/insets/InsetFloatList.h
index 489b0fe..ce6caa5 100644
--- a/src/insets/InsetFloatList.h
+++ b/src/insets/InsetFloatList.h
@@ -32,7 +32,7 @@ public:
        ///
        InsetCode lyxCode() const override { return FLOAT_LIST_CODE; }
        ///
-       RowFlags rowFlags() const override { return Display; }
+       int rowFlags() const override { return Display; }
        ///
        void write(std::ostream &) const override;
        ///
diff --git a/src/insets/InsetInclude.cpp b/src/insets/InsetInclude.cpp
index aeae2fb..10ea52b 100644
--- a/src/insets/InsetInclude.cpp
+++ b/src/insets/InsetInclude.cpp
@@ -1251,7 +1251,7 @@ string InsetInclude::contextMenuName() const
 }
 
 
-Inset::RowFlags InsetInclude::rowFlags() const
+int InsetInclude::rowFlags() const
 {
        return type(params()) == INPUT ? Inline : Display;
 }
diff --git a/src/insets/InsetInclude.h b/src/insets/InsetInclude.h
index 8585222..d62c751 100644
--- a/src/insets/InsetInclude.h
+++ b/src/insets/InsetInclude.h
@@ -75,7 +75,7 @@ public:
        ///
        void draw(PainterInfo & pi, int x, int y) const override;
        ///
-       RowFlags rowFlags() const override;
+       int rowFlags() const override;
        ///
        InsetCode lyxCode() const override { return INCLUDE_CODE; }
        ///
diff --git a/src/insets/InsetIndex.h b/src/insets/InsetIndex.h
index bddb8ba..b064cc7 100644
--- a/src/insets/InsetIndex.h
+++ b/src/insets/InsetIndex.h
@@ -119,7 +119,7 @@ public:
        ///
        bool hasSettings() const override;
        ///
-       RowFlags rowFlags() const override { return Display; }
+       int rowFlags() const override { return Display; }
        //@}
 
        /// \name Static public methods obligated for InsetCommand derived 
classes
diff --git a/src/insets/InsetListings.cpp b/src/insets/InsetListings.cpp
index e8fe8b1..57df06e 100644
--- a/src/insets/InsetListings.cpp
+++ b/src/insets/InsetListings.cpp
@@ -64,7 +64,7 @@ InsetListings::~InsetListings()
 }
 
 
-Inset::RowFlags InsetListings::rowFlags() const
+int InsetListings::rowFlags() const
 {
        return params().isInline() || params().isFloat() ? Inline : Display | 
AlignLeft;
 }
diff --git a/src/insets/InsetListings.h b/src/insets/InsetListings.h
index 41be439..9d4eeb1 100644
--- a/src/insets/InsetListings.h
+++ b/src/insets/InsetListings.h
@@ -46,7 +46,7 @@ private:
        ///
        InsetCode lyxCode() const override { return LISTINGS_CODE; }
        /// lstinline is inlined, normal listing is displayed
-       RowFlags rowFlags() const override;
+       int rowFlags() const override;
        ///
        docstring layoutName() const override;
        ///
diff --git a/src/insets/InsetNewline.h b/src/insets/InsetNewline.h
index 3d540a8..1ef0ae5 100644
--- a/src/insets/InsetNewline.h
+++ b/src/insets/InsetNewline.h
@@ -47,7 +47,7 @@ public:
        explicit InsetNewline(InsetNewlineParams par) : Inset(0)
        { params_.kind = par.kind; }
        ///
-       RowFlags rowFlags() const override { return BreakAfter | RowAfter; }
+       int rowFlags() const override { return AlwaysBreakAfter; }
        ///
        static void string2params(std::string const &, InsetNewlineParams &);
        ///
diff --git a/src/insets/InsetNewpage.h b/src/insets/InsetNewpage.h
index f020488..d086276 100644
--- a/src/insets/InsetNewpage.h
+++ b/src/insets/InsetNewpage.h
@@ -76,7 +76,7 @@ private:
        ///
        void write(std::ostream & os) const override;
        ///
-       RowFlags rowFlags() const override { return (params_.kind == 
InsetNewpageParams::NOPAGEBREAK) ? Inline : Display; }
+       int rowFlags() const override { return (params_.kind == 
InsetNewpageParams::NOPAGEBREAK) ? Inline : Display; }
        ///
        docstring insetLabel() const;
        ///
diff --git a/src/insets/InsetNomencl.h b/src/insets/InsetNomencl.h
index 1778e01..362cd46 100644
--- a/src/insets/InsetNomencl.h
+++ b/src/insets/InsetNomencl.h
@@ -94,7 +94,7 @@ public:
        ///
        bool hasSettings() const override { return true; }
        ///
-       RowFlags rowFlags() const override { return Display; }
+       int rowFlags() const override { return Display; }
        ///
        void latex(otexstream &, OutputParams const &) const override;
        ///
diff --git a/src/insets/InsetSeparator.h b/src/insets/InsetSeparator.h
index f7e0ab9..9352bdf 100644
--- a/src/insets/InsetSeparator.h
+++ b/src/insets/InsetSeparator.h
@@ -65,7 +65,7 @@ public:
                return docstring();
        }
        ///
-       RowFlags rowFlags() const override { return BreakAfter; }
+       int rowFlags() const override { return BreakAfter; }
 private:
        ///
        InsetCode lyxCode() const override { return SEPARATOR_CODE; }
diff --git a/src/insets/InsetSpace.cpp b/src/insets/InsetSpace.cpp
index 9585596..1a2eb09 100644
--- a/src/insets/InsetSpace.cpp
+++ b/src/insets/InsetSpace.cpp
@@ -192,7 +192,7 @@ bool InsetSpace::getStatus(Cursor & cur, FuncRequest const 
& cmd,
 }
 
 
-Inset::RowFlags InsetSpace::rowFlags() const
+int InsetSpace::rowFlags() const
 {
        switch (params_.kind) {
                case InsetSpaceParams::PROTECTED:
diff --git a/src/insets/InsetSpace.h b/src/insets/InsetSpace.h
index 5cf7aa7..e401d6d 100644
--- a/src/insets/InsetSpace.h
+++ b/src/insets/InsetSpace.h
@@ -115,7 +115,7 @@ public:
        ///
        docstring toolTip(BufferView const & bv, int x, int y) const override;
        /// unprotected spaces allow line breaking after them
-       RowFlags rowFlags() const override;
+       int rowFlags() const override;
        ///
        void metrics(MetricsInfo &, Dimension &) const override;
        ///
diff --git a/src/insets/InsetSpecialChar.cpp b/src/insets/InsetSpecialChar.cpp
index 3ecdaf1..88af653 100644
--- a/src/insets/InsetSpecialChar.cpp
+++ b/src/insets/InsetSpecialChar.cpp
@@ -83,7 +83,7 @@ docstring InsetSpecialChar::toolTip(BufferView const &, int, 
int) const
 }
 
 
-Inset::RowFlags InsetSpecialChar::rowFlags() const
+int InsetSpecialChar::rowFlags() const
 {
        switch (kind_) {
        case ALLOWBREAK:
diff --git a/src/insets/InsetSpecialChar.h b/src/insets/InsetSpecialChar.h
index 3056b10..0c8cc36 100644
--- a/src/insets/InsetSpecialChar.h
+++ b/src/insets/InsetSpecialChar.h
@@ -63,7 +63,7 @@ public:
        ///
        docstring toolTip(BufferView const & bv, int x, int y) const override;
        /// some special chars allow line breaking after them
-       RowFlags rowFlags() const override;
+       int rowFlags() const override;
        ///
        void metrics(MetricsInfo &, Dimension &) const override;
        ///
diff --git a/src/insets/InsetTOC.h b/src/insets/InsetTOC.h
index 045ae07..ca3f463 100644
--- a/src/insets/InsetTOC.h
+++ b/src/insets/InsetTOC.h
@@ -37,7 +37,7 @@ public:
        ///
        docstring layoutName() const override;
        ///
-       RowFlags rowFlags() const override { return Display; }
+       int rowFlags() const override { return Display; }
        ///
        void validate(LaTeXFeatures &) const override;
        ///
diff --git a/src/insets/InsetTabular.cpp b/src/insets/InsetTabular.cpp
index fcf89e7..24bde65 100644
--- a/src/insets/InsetTabular.cpp
+++ b/src/insets/InsetTabular.cpp
@@ -6144,21 +6144,21 @@ bool InsetTabular::getStatus(Cursor & cur, FuncRequest 
const & cmd,
 }
 
 
-Inset::RowFlags InsetTabular::rowFlags() const
-{
-               if (tabular.is_long_tabular) {
-                       switch (tabular.longtabular_alignment) {
-                       case Tabular::LYX_LONGTABULAR_ALIGN_LEFT:
-                               return Display | AlignLeft;
-                       case Tabular::LYX_LONGTABULAR_ALIGN_CENTER:
-                               return Display;
-                       case Tabular::LYX_LONGTABULAR_ALIGN_RIGHT:
-                               return Display | AlignRight;
-                       default:
-                               return Display;
-                       }
-               } else
-                       return Inline;
+int InsetTabular::rowFlags() const
+{
+       if (tabular.is_long_tabular) {
+               switch (tabular.longtabular_alignment) {
+               case Tabular::LYX_LONGTABULAR_ALIGN_LEFT:
+                       return Display | AlignLeft;
+               case Tabular::LYX_LONGTABULAR_ALIGN_CENTER:
+                       return Display;
+               case Tabular::LYX_LONGTABULAR_ALIGN_RIGHT:
+                       return Display | AlignRight;
+               default:
+                       return Display;
+               }
+       } else
+               return Inline;
 }
 
 
diff --git a/src/insets/InsetTabular.h b/src/insets/InsetTabular.h
index 8d1be1b..0d4e7af 100644
--- a/src/insets/InsetTabular.h
+++ b/src/insets/InsetTabular.h
@@ -989,7 +989,7 @@ public:
        //
        bool isTable() const override { return true; }
        ///
-       RowFlags rowFlags() const override;
+       int rowFlags() const override;
        ///
        void latex(otexstream &, OutputParams const &) const override;
        ///
diff --git a/src/insets/InsetVSpace.h b/src/insets/InsetVSpace.h
index 9b95f00..51c2b5b 100644
--- a/src/insets/InsetVSpace.h
+++ b/src/insets/InsetVSpace.h
@@ -62,7 +62,7 @@ private:
        ///
        void write(std::ostream & os) const override;
        ///
-       RowFlags rowFlags() const override { return Display; }
+       int rowFlags() const override { return Display; }
        ///
        void doDispatch(Cursor & cur, FuncRequest & cmd) override;
        ///
diff --git a/src/mathed/InsetMathHull.cpp b/src/mathed/InsetMathHull.cpp
index 97d95da..26ac40b 100644
--- a/src/mathed/InsetMathHull.cpp
+++ b/src/mathed/InsetMathHull.cpp
@@ -988,7 +988,7 @@ bool InsetMathHull::outerDisplay() const
 }
 
 
-Inset::RowFlags InsetMathHull::rowFlags() const
+int InsetMathHull::rowFlags() const
 {
        switch (type_) {
        case hullUnknown:
diff --git a/src/mathed/InsetMathHull.h b/src/mathed/InsetMathHull.h
index 0b865be..b019188 100644
--- a/src/mathed/InsetMathHull.h
+++ b/src/mathed/InsetMathHull.h
@@ -288,7 +288,7 @@ public:
        ///
        Inset * editXY(Cursor & cur, int x, int y) override;
        ///
-       RowFlags rowFlags() const override;
+       int rowFlags() const override;
        /// helper function
        bool display() const { return rowFlags() & Display; }
 

commit de9b121be154503445f7a07acdc01701a669dfc7
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Sun Jul 11 15:19:37 2021 +0200

    Create new method TM::tokenizeParagraph
    
    This contains large parts of breakRow, but creates a unique row for the 
paragraph.
    
    The parts taken or not in redoParagraph are annotated.
    
    The new method is not used yet.

diff --git a/src/TextMetrics.cpp b/src/TextMetrics.cpp
index a2a3e4b..f14fe1f 100644
--- a/src/TextMetrics.cpp
+++ b/src/TextMetrics.cpp
@@ -868,21 +868,142 @@ private:
 
 } // namespace
 
+
+Row TextMetrics::tokenizeParagraph(pit_type const pit) const
+{
+       Row row;
+       row.pit(pit);
+       Paragraph const & par = text_->getPar(pit);
+       Buffer const & buf = text_->inset().buffer();
+       BookmarksSection::BookmarkPosList bpl =
+               theSession().bookmarks().bookmarksInPar(buf.fileName(), 
par.id());
+
+       pos_type const end = par.size();
+       pos_type const body_pos = par.beginOfBody();
+
+       // check for possible inline completion
+       DocIterator const & ic_it = bv_->inlineCompletionPos();
+       pos_type ic_pos = -1;
+       if (ic_it.inTexted() && ic_it.text() == text_ && ic_it.pit() == pit)
+               ic_pos = ic_it.pos();
+
+       // Now we iterate through until we reach the right margin
+       // or the end of the par, then build a representation of the row.
+       pos_type i = 0;
+       FontIterator fi = FontIterator(*this, par, pit, 0);
+       // The real stopping condition is a few lines below.
+       while (true) {
+               // Firstly, check whether there is a bookmark here.
+               if (lyxrc.bookmarks_visibility == LyXRC::BMK_INLINE)
+                       for (auto const & bp_p : bpl)
+                               if (bp_p.second == i) {
+                                       Font f = *fi;
+                                       f.fontInfo().setColor(Color_bookmark);
+                                       // ❶ U+2776 DINGBAT NEGATIVE CIRCLED 
DIGIT ONE
+                                       char_type const ch = 0x2775 + 
bp_p.first;
+                                       row.addVirtual(i, docstring(1, ch), f, 
Change());
+                               }
+
+               // The stopping condition is here so that the display of a
+               // bookmark can take place at paragraph start too.
+               if (i >= end)
+                       break;
+
+               char_type c = par.getChar(i);
+               // The most special cases are handled first.
+               if (par.isInset(i)) {
+                       Inset const * ins = par.getInset(i);
+                       Dimension dim = bv_->coordCache().insets().dim(ins);
+                       row.add(i, ins, dim, *fi, par.lookupChange(i));
+               } else if (c == ' ' && i + 1 == body_pos) {
+                       // There is a space at i, but it should not be
+                       // added as a separator, because it is just
+                       // before body_pos. Instead, insert some spacing to
+                       // align text
+                       FontMetrics const & fm = 
theFontMetrics(text_->labelFont(par));
+                       // this is needed to make sure that the row width is 
correct
+                       row.finalizeLast();
+                       int const add = max(fm.width(par.layout().labelsep),
+                                           labelEnd(pit) - row.width());
+                       row.addSpace(i, add, *fi, par.lookupChange(i));
+               } else if (c == '\t')
+                       row.addSpace(i, theFontMetrics(*fi).width(from_ascii("  
  ")),
+                                    *fi, par.lookupChange(i));
+               else if (c == 0x2028 || c == 0x2029) {
+                       /**
+                        * U+2028 LINE SEPARATOR
+                        * U+2029 PARAGRAPH SEPARATOR
+
+                        * These are special unicode characters that break
+                        * lines/pragraphs. Not handling them lead to trouble 
wrt
+                        * Qt QTextLayout formatting. We add a visible character
+                        * on screen so that the user can see that something is
+                        * happening.
+                       */
+                       row.finalizeLast();
+                       // ⤶ U+2936 ARROW POINTING DOWNWARDS THEN CURVING 
LEFTWARDS
+                       // ¶ U+00B6 PILCROW SIGN
+                       char_type const screen_char = (c == 0x2028) ? 0x2936 : 
0x00B6;
+                       row.add(i, screen_char, *fi, par.lookupChange(i));
+               } else
+                       row.add(i, c, *fi, par.lookupChange(i));
+
+               // add inline completion width
+               // draw logically behind the previous character
+               if (ic_pos == i + 1 && !bv_->inlineCompletion().empty()) {
+                       docstring const comp = bv_->inlineCompletion();
+                       size_t const uniqueTo 
=bv_->inlineCompletionUniqueChars();
+                       Font f = *fi;
+
+                       if (uniqueTo > 0) {
+                               f.fontInfo().setColor(Color_inlinecompletion);
+                               row.addVirtual(i + 1, comp.substr(0, uniqueTo), 
f, Change());
+                       }
+                       f.fontInfo().setColor(Color_nonunique_inlinecompletion);
+                       row.addVirtual(i + 1, comp.substr(uniqueTo), f, 
Change());
+               }
+
+               ++i;
+               ++fi;
+       }
+       row.finalizeLast();
+       row.endpos(end);
+
+       // End of paragraph marker. The logic here is almost the
+       // same as in redoParagraph, remember keep them in sync.
+       ParagraphList const & pars = text_->paragraphs();
+       Change const & change = par.lookupChange(i);
+       if ((lyxrc.paragraph_markers || change.changed())
+           && i == end && size_type(pit + 1) < pars.size()) {
+               // add a virtual element for the end-of-paragraph
+               // marker; it is shown on screen, but does not exist
+               // in the paragraph.
+               Font f(text_->layoutFont(pit));
+               f.fontInfo().setColor(Color_paragraphmarker);
+               f.setLanguage(par.getParLanguage(buf.params()));
+               // ¶ U+00B6 PILCROW SIGN
+               row.addVirtual(end, docstring(1, char_type(0x00B6)), f, change);
+       }
+
+       return row;
+}
+
+
 /** This is the function where the hard work is done. The code here is
  * very sensitive to small changes :) Note that part of the
  * intelligence is also in Row::shortenIfNeeded.
  */
 bool TextMetrics::breakRow(Row & row, int const right_margin) const
 {
-       LATTEST(row.empty());
-       Paragraph const & par = text_->getPar(row.pit());
-       Buffer const & buf = text_->inset().buffer();
-       BookmarksSection::BookmarkPosList bpl =
-               theSession().bookmarks().bookmarksInPar(buf.fileName(), 
par.id());
+       LATTEST(row.empty());//
+       Paragraph const & par = text_->getPar(row.pit());//
+       Buffer const & buf = text_->inset().buffer();//
+       BookmarksSection::BookmarkPosList bpl =//
+               theSession().bookmarks().bookmarksInPar(buf.fileName(), 
par.id());//
 
-       pos_type const end = par.size();
+       pos_type const end = par.size();//
        pos_type const pos = row.pos();
-       pos_type const body_pos = par.beginOfBody();
+       pos_type const body_pos = par.beginOfBody();//
        bool const is_rtl = text_->isRTL(row.pit());
        bool need_new_row = false;
 
@@ -897,14 +1018,14 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
        int const width = max_width_ - row.right_margin;
 
        // check for possible inline completion
-       DocIterator const & ic_it = bv_->inlineCompletionPos();
-       pos_type ic_pos = -1;
-       if (ic_it.inTexted() && ic_it.text() == text_ && ic_it.pit() == 
row.pit())
-               ic_pos = ic_it.pos();
+       DocIterator const & ic_it = bv_->inlineCompletionPos();//
+       pos_type ic_pos = -1;//
+       if (ic_it.inTexted() && ic_it.text() == text_ && ic_it.pit() == 
row.pit())//
+               ic_pos = ic_it.pos();//
 
        // Now we iterate through until we reach the right margin
        // or the end of the par, then build a representation of the row.
-       pos_type i = pos;
+       pos_type i = 
pos;//---------------------------------------------------vvv
        FontIterator fi = FontIterator(*this, par, row.pit(), pos);
        // The real stopping condition is a few lines below.
        while (true) {
@@ -921,7 +1042,7 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
 
                // The stopping condition is here so that the display of a
                // bookmark can take place at paragraph start too.
-               if (i >= end || (i != pos && row.width() > width))
+               if (i >= end || (i != pos && row.width() > width))//^width
                        break;
 
                char_type c = par.getChar(i);
@@ -976,8 +1097,9 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
                        }
                        f.fontInfo().setColor(Color_nonunique_inlinecompletion);
                        row.addVirtual(i + 1, comp.substr(uniqueTo), f, 
Change());
-               }
+               
}//---------------------------------------------------------------^^^
 
+               // FIXME: Handle when breaking the rows
                // Handle some situations that abruptly terminate the row
                // - Before an inset with BreakBefore
                // - After an inset with BreakAfter
@@ -1002,6 +1124,7 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
                ++i;
                ++fi;
        }
+       
//--------------------------------------------------------------------vvv
        row.finalizeLast();
        row.endpos(i);
 
@@ -1010,7 +1133,7 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
        ParagraphList const & pars = text_->paragraphs();
        Change const & change = par.lookupChange(i);
        if ((lyxrc.paragraph_markers || change.changed())
-           && !need_new_row
+           && !need_new_row // not this
            && i == end && size_type(row.pit() + 1) < pars.size()) {
                // add a virtual element for the end-of-paragraph
                // marker; it is shown on screen, but does not exist
@@ -1025,6 +1148,8 @@ bool TextMetrics::breakRow(Row & row, int const 
right_margin) const
        // Is there a end-of-paragaph change?
        if (i == end && par.lookupChange(end).changed() && !need_new_row)
                row.needsChangeBar(true);
+    //--------------------------------------------------------------------^^^
+       // FIXME : nothing below this
 
        // if the row is too large, try to cut at last separator. In case
        // of success, reset indication that the row was broken abruptly.
diff --git a/src/TextMetrics.h b/src/TextMetrics.h
index 1501250..1666cd0 100644
--- a/src/TextMetrics.h
+++ b/src/TextMetrics.h
@@ -151,6 +151,8 @@ private:
        /// FIXME??
        int labelEnd(pit_type const pit) const;
 
+       Row tokenizeParagraph(pit_type pit) const;
+
        /// sets row.end to the pos value *after* which a row should break.
        /// for example, the pos after which isNewLine(pos) == true
        /// \return true when another row is required (after a newline)

commit 8a45ae5415c34e69b1778d51b890ce46d47caa5a
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Sun Jul 11 15:33:33 2021 +0200

    Small Row cleanups
    
    Move declaration of RowList to Row.h
    
    Move initialization of POD members of Row and Row::Element to declaration.
    
    Make method isVirtual() depend on type.
    
    Add new row element type INVALID and method isValid()
    
    Make methods R::E::left/right_pos inline.
    
    Add method R::E::splitAt() that returns an element containing the
    remaining stuff, or an invalid element if nothing was split. breakAt
    is now a simple wrapper around this function.
    
    Add method R::push_back().

diff --git a/src/ParagraphMetrics.h b/src/ParagraphMetrics.h
index 9889473..1d690aa 100644
--- a/src/ParagraphMetrics.h
+++ b/src/ParagraphMetrics.h
@@ -20,17 +20,8 @@
 #include "Dimension.h"
 #include "Row.h"
 
-#include <vector>
-
 namespace lyx {
 
-/**
- * Each paragraph is broken up into a number of rows on the screen.
- * This is a list of such on-screen rows, ordered from the top row
- * downwards.
- */
-typedef std::vector<Row> RowList;
-
 class BufferView;
 class Paragraph;
 
diff --git a/src/Row.cpp b/src/Row.cpp
index 2f6db3d..da2d885 100644
--- a/src/Row.cpp
+++ b/src/Row.cpp
@@ -124,53 +124,43 @@ pos_type Row::Element::x2pos(int &x) const
                        x = 0;
                        i = isRTL();
                }
+               break;
+       case INVALID:
+               LYXERR0("x2pos: INVALID row element !");
        }
        //lyxerr << "=> p=" << pos + i << " x=" << x << endl;
        return pos + i;
 }
 
 
-bool Row::Element::breakAt(int w, bool force)
+Row::Element Row::Element::splitAt(int w, bool force)
 {
        if (type != STRING)
-               return false;
+               return Element();
 
        FontMetrics const & fm = theFontMetrics(font);
        dim.wid = w;
        int const i = fm.breakAt(str, dim.wid, isRTL(), force);
        if (i != -1) {
+               Element ret(STRING, pos + i, font, change);
+               ret.str = str.substr(i);
+               ret.endpos = ret.pos + ret.str.length();
                str.erase(i);
                endpos = pos + i;
                //lyxerr << "breakAt(" << w << ")  Row element Broken at " << x 
<< "(w(str)=" << fm.width(str) << "): e=" << *this << endl;
+               return ret;
        }
 
-       return i != - 1;
+       return Element();
 }
 
 
-pos_type Row::Element::left_pos() const
-{
-       return isRTL() ? endpos : pos;
-}
-
-
-pos_type Row::Element::right_pos() const
+bool Row::Element::breakAt(int w, bool force)
 {
-       return isRTL() ? pos : endpos;
+       return splitAt(w, force).isValid();
 }
 
 
-Row::Row()
-       : separator(0), label_hfill(0), left_margin(0), right_margin(0),
-         sel_beg(-1), sel_end(-1),
-         begin_margin_sel(false), end_margin_sel(false),
-         changed_(true),
-         pit_(0), pos_(0), end_(0),
-         right_boundary_(false), flushed_(false), rtl_(false),
-         changebar_(false)
-{}
-
-
 bool Row::isMarginSelected(bool left, DocIterator const & beg,
                DocIterator const & end) const
 {
@@ -262,6 +252,9 @@ ostream & operator<<(ostream & os, Row::Element const & e)
        case Row::SPACE:
                os << "SPACE: ";
                break;
+       case Row::INVALID:
+               os << "INVALID: ";
+               break;
        }
        os << "width=" << e.full_width();
        return os;
@@ -443,6 +436,13 @@ void Row::addSpace(pos_type const pos, int const width,
 }
 
 
+void Row::push_back(Row::Element const & e)
+{
+       dim_.wid += e.dim.wid;
+       elements_.push_back(e);
+}
+
+
 void Row::pop_back()
 {
        dim_.wid -= elements_.back().dim.wid;
diff --git a/src/Row.h b/src/Row.h
index b54a233..3048cf1 100644
--- a/src/Row.h
+++ b/src/Row.h
@@ -49,7 +49,9 @@ public:
                // An inset
                INSET,
                // Some spacing described by its width, not a string
-               SPACE
+               SPACE,
+               // Something that should not happen (for error handling)
+               INVALID
        };
 
 /**
@@ -57,9 +59,12 @@ public:
  * by other methods that need to parse the Row contents.
  */
        struct Element {
+               //
+               Element() = default;
+               //
                Element(Type const t, pos_type p, Font const & f, Change const 
& ch)
-                       : type(t), pos(p), endpos(p + 1), inset(0),
-                         extra(0), font(f), change(ch), final(false) {}
+                       : type(t), pos(p), endpos(p + 1), font(f), change(ch) {}
+
 
                // Return the number of separator in the element (only STRING 
type)
                int countSeparators() const;
@@ -86,40 +91,49 @@ public:
                 *  adjusted to the actual pixel position.
                */
                pos_type x2pos(int &x) const;
+               /** Break the element in two if possible, so that its width is 
less
+                * than \param w.
+                * \return an element containing the remainder of the text, or
+                *   an invalid element if nothing happened.
+                * \param w: the desired maximum width
+                * \param force: if true, the string is cut at any place, 
otherwise it
+                *   respects the row breaking rules of characters.
+                */
+               Element splitAt(int w, bool force);
                /** Break the element if possible, so that its width is less
                 * than \param w. Returns true on success. When \param force
-                * is true, the string is cut at any place, other wise it
+                * is true, the string is cut at any place, otherwise it
                 * respects the row breaking rules of characters.
                 */
                bool breakAt(int w, bool force);
 
-               // Returns the position on left side of the element.
-               pos_type left_pos() const;
-               // Returns the position on right side of the element.
-               pos_type right_pos() const;
-
                //
                bool isRTL() const { return font.isVisibleRightToLeft(); }
                // This is true for virtual elements.
-               // Note that we do not use the type here. The two definitions
-               // should be equivalent
-               bool isVirtual() const { return pos == endpos; }
+               bool isVirtual() const { return type == VIRTUAL; }
+               // Invalid element, for error handling
+               bool isValid() const { return type !=INVALID; }
+
+               // Returns the position on left side of the element.
+               pos_type left_pos() const { return isRTL() ? endpos : pos; };
+               // Returns the position on right side of the element.
+               pos_type right_pos() const { return isRTL() ? pos : endpos; };
 
                // The kind of row element
-               Type type;
+               Type type = INVALID;
                // position of the element in the paragraph
-               pos_type pos;
+               pos_type pos = 0;
                // first position after the element in the paragraph
-               pos_type endpos;
+               pos_type endpos = 0;
                // The dimension of the chunk (does not contains the
                // separator correction)
                Dimension dim;
 
                // Non-zero only if element is an inset
-               Inset const * inset;
+               Inset const * inset = nullptr;
 
                // Only non-null for justified rows
-               double extra;
+               double extra = 0;
 
                // Non-empty if element is a string or is virtual
                docstring str;
@@ -128,14 +142,15 @@ public:
                //
                Change change;
                // is it possible to add contents to this element?
-               bool final;
+               bool final = false;
 
                friend std::ostream & operator<<(std::ostream & os, Element 
const & row);
        };
 
 
        ///
-       Row();
+       Row() {}
+
        /**
         * Helper function: set variable \c var to value \c val, and mark
         * row as changed is the values were different. This is intended
@@ -264,7 +279,9 @@ public:
        Element & back() { return elements_.back(); }
        ///
        Element const & back() const { return elements_.back(); }
-       /// remove last element
+       /// add element at the end and update width
+       void push_back(Element const &);
+       /// remove last element and update width
        void pop_back();
        /**
         * if row width is too large, remove all elements after last
@@ -301,21 +318,21 @@ public:
        friend std::ostream & operator<<(std::ostream & os, Row const & row);
 
        /// additional width for separators in justified rows (i.e. space)
-       double separator;
+       double separator = 0;
        /// width of hfills in the label
-       double label_hfill;
+       double label_hfill = 0;
        /// the left margin position of the row
-       int left_margin;
+       int left_margin = 0;
        /// the right margin of the row
-       int right_margin;
+       int right_margin = 0;
        ///
-       mutable pos_type sel_beg;
+       mutable pos_type sel_beg = -1;
        ///
-       mutable pos_type sel_end;
+       mutable pos_type sel_end = -1;
        ///
-       mutable bool begin_margin_sel;
+       mutable bool begin_margin_sel = false;
        ///
-       mutable bool end_margin_sel;
+       mutable bool end_margin_sel = false;
 
 private:
        /// Decides whether the margin is selected.
@@ -340,28 +357,35 @@ private:
        Elements elements_;
 
        /// has the Row appearance changed since last drawing?
-       mutable bool changed_;
+       mutable bool changed_ = true;
        /// Index of the paragraph that contains this row
-       pit_type pit_;
+       pit_type pit_ = 0;
        /// first pos covered by this row
-       pos_type pos_;
+       pos_type pos_ = 0;
        /// one behind last pos covered by this row
-       pos_type end_;
+       pos_type end_ = 0;
        // Is there a boundary at the end of the row (display inset...)
-       bool right_boundary_;
+       bool right_boundary_ = false;
        // Shall the row be flushed when it is supposed to be justified?
-       bool flushed_;
+       bool flushed_ = false;
        /// Row dimension.
        Dimension dim_;
        /// Row contents dimension. Does not contain the space above/below row.
        Dimension contents_dim_;
        /// true when this row lives in a right-to-left paragraph
-       bool rtl_;
+       bool rtl_ = false;
        /// true when a changebar should be drawn in the margin
-       bool changebar_;
+       bool changebar_ = false;
 };
 
 
+/**
+ * Each paragraph is broken up into a number of rows on the screen.
+ * This is a list of such on-screen rows, ordered from the top row
+ * downwards.
+ */
+typedef std::vector<Row> RowList;
+
 } // namespace lyx
 
 #endif
diff --git a/src/RowPainter.cpp b/src/RowPainter.cpp
index 400b7b6..656f89a 100644
--- a/src/RowPainter.cpp
+++ b/src/RowPainter.cpp
@@ -565,6 +565,10 @@ void RowPainter::paintText()
 
                case Row::SPACE:
                        paintTextDecoration(e);
+                       break;
+
+               case Row::INVALID:
+                       LYXERR0("Trying to paint INVALID row element.");
                }
 
                // The markings of foreign languages

commit 3d44008bb10c53fcd824f3c2af1ca178ecd504b0
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Sat Jul 10 23:21:27 2021 +0200

    Change FontMetrics::breakAt to return a position
    
    Since we intend to break the row element in two, it is not good to
    truncate the string too early.
    
    Moreover, the row element width is now set at this point, even if no
    breaking occurs.

diff --git a/src/Row.cpp b/src/Row.cpp
index 16ae9ae..2f6db3d 100644
--- a/src/Row.cpp
+++ b/src/Row.cpp
@@ -132,19 +132,19 @@ pos_type Row::Element::x2pos(int &x) const
 
 bool Row::Element::breakAt(int w, bool force)
 {
-       if (type != STRING || dim.wid <= w)
+       if (type != STRING)
                return false;
 
        FontMetrics const & fm = theFontMetrics(font);
-       int x = w;
-       if(fm.breakAt(str, x, isRTL(), force)) {
-               dim.wid = x;
-               endpos = pos + str.length();
+       dim.wid = w;
+       int const i = fm.breakAt(str, dim.wid, isRTL(), force);
+       if (i != -1) {
+               str.erase(i);
+               endpos = pos + i;
                //lyxerr << "breakAt(" << w << ")  Row element Broken at " << x 
<< "(w(str)=" << fm.width(str) << "): e=" << *this << endl;
-               return true;
        }
 
-       return false;
+       return i != - 1;
 }
 
 
diff --git a/src/frontends/FontMetrics.h b/src/frontends/FontMetrics.h
index 2a6ffea..b562ebf 100644
--- a/src/frontends/FontMetrics.h
+++ b/src/frontends/FontMetrics.h
@@ -122,13 +122,14 @@ public:
         */
        virtual int x2pos(docstring const & s, int & x, bool rtl, double ws) 
const = 0;
        /**
-        * Break string at width at most x.
-        * \return true if successful
+        * Break string s at width at most x.
+        * \return break position (-1 if not successful)
+        * \param position x is updated to real width
         * \param rtl is true for right-to-left layout
         * \param force is false for breaking at word separator, true for
         *   arbitrary position.
         */
-       virtual bool breakAt(docstring & s, int & x, bool rtl, bool force) 
const = 0;
+       virtual int breakAt(docstring const & s, int & x, bool rtl, bool force) 
const = 0;
        /// return char dimension for the font.
        virtual Dimension const dimension(char_type c) const = 0;
        /**
diff --git a/src/frontends/qt/GuiFontMetrics.cpp 
b/src/frontends/qt/GuiFontMetrics.cpp
index 5e8f535..25bf7a4 100644
--- a/src/frontends/qt/GuiFontMetrics.cpp
+++ b/src/frontends/qt/GuiFontMetrics.cpp
@@ -533,7 +533,7 @@ GuiFontMetrics::breakAt_helper(docstring const & s, int 
const x,
        tl.endLayout();
        int const line_wid = iround(line.horizontalAdvance());
        if ((force && line.textLength() == offset) || line_wid > x)
-               return {-1, -1};
+               return {-1, line_wid};
        /* Since QString is UTF-16 and docstring is UCS-4, the offsets may
         * not be the same when there are high-plan unicode characters
         * (bug #10443).
@@ -557,6 +557,9 @@ GuiFontMetrics::breakAt_helper(docstring const & s, int 
const x,
                --len;
        LASSERT(len > 0 || qlen == 0, /**/);
 #endif
+       // si la chaîne est déjà trop courte, on ne coupe pas
+       if (len == static_cast<int>(s.length()))
+               len = -1;
        return {len, line_wid};
 }
 
@@ -568,7 +571,7 @@ uint qHash(BreakAtKey const & key)
 }
 
 
-bool GuiFontMetrics::breakAt(docstring & s, int & x, bool const rtl, bool 
const force) const
+int GuiFontMetrics::breakAt(docstring const & s, int & x, bool const rtl, bool 
const force) const
 {
        PROFILE_THIS_BLOCK(breakAt);
        if (s.empty())
@@ -583,11 +586,8 @@ bool GuiFontMetrics::breakAt(docstring & s, int & x, bool 
const rtl, bool const
                pp = breakAt_helper(s, x, rtl, force);
                breakat_cache_.insert(key, pp, sizeof(key) + s.size() * 
sizeof(char_type));
        }
-       if (pp.first == -1)
-               return false;
-       s = s.substr(0, pp.first);
        x = pp.second;
-       return true;
+       return pp.first;
 }
 
 
diff --git a/src/frontends/qt/GuiFontMetrics.h 
b/src/frontends/qt/GuiFontMetrics.h
index 9c7ce89..ef8588a 100644
--- a/src/frontends/qt/GuiFontMetrics.h
+++ b/src/frontends/qt/GuiFontMetrics.h
@@ -77,7 +77,7 @@ public:
        int signedWidth(docstring const & s) const override;
        int pos2x(docstring const & s, int pos, bool rtl, double ws) const 
override;
        int x2pos(docstring const & s, int & x, bool rtl, double ws) const 
override;
-       bool breakAt(docstring & s, int & x, bool rtl, bool force) const 
override;
+       int breakAt(docstring const & s, int & x, bool rtl, bool force) const 
override;
        Dimension const dimension(char_type c) const override;
 
        void rectText(docstring const & str,

commit a5f1b158589fc9d3a5d1f8291461d0001d06ea09
Author: Jean-Marc Lasgouttes <[email protected]>
Date:   Sat Oct 9 12:25:29 2021 +0200

    Make CoordCache assertions less annoying.

diff --git a/src/CoordCache.h b/src/CoordCache.h
index b99cbe6..f26b6b2 100644
--- a/src/CoordCache.h
+++ b/src/CoordCache.h
@@ -80,13 +80,13 @@ public:
 
        Geometry & geometry(T const * thing)
        {
-               check(thing, "geometry");
+               checkDim(thing, "geometry");
                return data_.find(thing)->second;
        }
 
        Geometry const & geometry(T const * thing) const
        {
-               check(thing, "geometry");
+               checkDim(thing, "geometry");
                return data_.find(thing)->second;
        }
 

-----------------------------------------------------------------------

Summary of changes:
 src/CoordCache.h |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)


hooks/post-receive
-- 
Repository for new features

-- 
lyx-cvs mailing list
[email protected]
http://lists.lyx.org/mailman/listinfo/lyx-cvs

Reply via email to