src/hb-buffer-private.hh | 4 src/hb-buffer.cc | 46 ++ src/hb-ot-layout-common-private.hh | 1 src/hb-ot-layout-gsub-table.hh | 1 src/hb-ot-layout-gsubgpos-private.hh | 157 ++++------ src/hb-ot-layout.cc | 1 test/shaping/Makefile.am | 10 test/shaping/fonts/sha1sum/4cce528e99f600ed9c25a2b69e32eb94a03b4ae8.ttf |binary test/shaping/fonts/sha1sum/ceadd106a8205214fbe7337ef9de32a862b59762.ttf |binary test/shaping/fonts/sha1sum/d629e7fedc0b350222d7987345fe61613fa3929a.ttf |binary test/shaping/fonts/sha1sum/f499fbc23865022234775c43503bba2e63978fe1.ttf |binary test/shaping/hb_test_tools.py | 2 test/shaping/run-tests.sh | 34 ++ test/shaping/tests/MANIFEST | 1 test/shaping/tests/context-matching.tests | 4 15 files changed, 177 insertions(+), 84 deletions(-)
New commits: commit 6b65a76b40522a4f57a6fedcbdfc5a4d736f1d3c Author: Behdad Esfahbod <[email protected]> Date: Mon Oct 14 18:51:39 2013 +0200 [otlayout] Fix (Chain)Context recursion! Previously we only supported recursive sublookups with ascending indices. We were also not correctly handling non-1-to-1 recursed lookups. Fix all that! Fixes the three tests in test/shaping/tests/context-matching.tests, which were derived from NotoSansBengali and NotoSansDevanagari among others. diff --git a/src/hb-buffer-private.hh b/src/hb-buffer-private.hh index a8cf770..7032390 100644 --- a/src/hb-buffer-private.hh +++ b/src/hb-buffer-private.hh @@ -103,6 +103,8 @@ struct hb_buffer_t { inline unsigned int backtrack_len (void) const { return have_output? out_len : idx; } + inline unsigned int lookahead_len (void) const + { return len - idx; } inline unsigned int next_serial (void) { return serial++; } HB_INTERNAL void allocate_var (unsigned int byte_i, unsigned int count, const char *owner); @@ -134,6 +136,7 @@ struct hb_buffer_t { HB_INTERNAL void output_info (const hb_glyph_info_t &glyph_info); /* Copies glyph at idx to output but doesn't advance idx */ HB_INTERNAL void copy_glyph (void); + HB_INTERNAL bool move_to (unsigned int i); /* i is output-buffer index. */ /* Copies glyph at idx to output and advance idx. * If there's no output, just advance idx. */ inline void @@ -181,6 +184,7 @@ struct hb_buffer_t { { return likely (size < allocated) ? true : enlarge (size); } HB_INTERNAL bool make_room_for (unsigned int num_in, unsigned int num_out); + HB_INTERNAL bool shift_forward (unsigned int count); HB_INTERNAL void *get_scratch_buffer (unsigned int *size); diff --git a/src/hb-buffer.cc b/src/hb-buffer.cc index 54626db..b778abb 100644 --- a/src/hb-buffer.cc +++ b/src/hb-buffer.cc @@ -139,6 +139,19 @@ hb_buffer_t::make_room_for (unsigned int num_in, return true; } +bool +hb_buffer_t::shift_forward (unsigned int count) +{ + assert (have_output); + if (unlikely (!ensure (len + count))) return false; + + memmove (info + idx + count, info + idx, (len - idx) * sizeof (info[0])); + len += count; + idx += count; + + return true; +} + void * hb_buffer_t::get_scratch_buffer (unsigned int *size) { @@ -345,6 +358,39 @@ hb_buffer_t::copy_glyph (void) out_len++; } +bool +hb_buffer_t::move_to (unsigned int i) +{ + if (!have_output) + { + assert (i <= len); + idx = i; + } + else if (out_len < i) + { + unsigned int count = i - out_len; + if (unlikely (!make_room_for (count, count))) return false; + + memmove (out_info + out_len, info + idx, count * sizeof (out_info[0])); + idx += count; + out_len += count; + } + else if (out_len > i) + { + /* Tricky part: rewinding... */ + unsigned int count = out_len - i; + + if (unlikely (idx < count && !shift_forward (count + 32))) return false; + + assert (idx >= count); + + idx -= count; + out_len -= count; + memmove (info + idx, out_info + out_len, count * sizeof (out_info[0])); + } + return true; +} + void hb_buffer_t::replace_glyph (hb_codepoint_t glyph_index) { diff --git a/src/hb-ot-layout-common-private.hh b/src/hb-ot-layout-common-private.hh index 2f6e804..8ef8424 100644 --- a/src/hb-ot-layout-common-private.hh +++ b/src/hb-ot-layout-common-private.hh @@ -39,6 +39,7 @@ namespace OT { #define NOT_COVERED ((unsigned int) -1) #define MAX_NESTING_LEVEL 8 +#define MAX_CONTEXT_LENGTH 32 diff --git a/src/hb-ot-layout-gsub-table.hh b/src/hb-ot-layout-gsub-table.hh index fefc71e..1588580 100644 --- a/src/hb-ot-layout-gsub-table.hh +++ b/src/hb-ot-layout-gsub-table.hh @@ -672,6 +672,7 @@ struct Ligature match_glyph, NULL, &end_offset, + NULL, &is_mark_ligature, &total_component_count))) return TRACE_RETURN (false); diff --git a/src/hb-ot-layout-gsubgpos-private.hh b/src/hb-ot-layout-gsubgpos-private.hh index 316f506..5366550 100644 --- a/src/hb-ot-layout-gsubgpos-private.hh +++ b/src/hb-ot-layout-gsubgpos-private.hh @@ -753,11 +753,14 @@ static inline bool match_input (hb_apply_context_t *c, match_func_t match_func, const void *match_data, unsigned int *end_offset = NULL, + unsigned int match_positions[MAX_CONTEXT_LENGTH] = NULL, bool *p_is_mark_ligature = NULL, unsigned int *p_total_component_count = NULL) { TRACE_APPLY (NULL); + if (unlikely (count > MAX_CONTEXT_LENGTH)) TRACE_RETURN (false); + hb_apply_context_t::skipping_forward_iterator_t skippy_iter (c, c->buffer->idx, count - 1); skippy_iter.set_match_func (match_func, match_data, input); if (skippy_iter.has_no_chance ()) return TRACE_RETURN (false); @@ -788,9 +791,13 @@ static inline bool match_input (hb_apply_context_t *c, unsigned int first_lig_id = get_lig_id (c->buffer->cur()); unsigned int first_lig_comp = get_lig_comp (c->buffer->cur()); + if (match_positions) + match_positions[0] = c->buffer->idx; for (unsigned int i = 1; i < count; i++) { if (!skippy_iter.next ()) return TRACE_RETURN (false); + if (match_positions) + match_positions[i] = skippy_iter.idx; unsigned int this_lig_id = get_lig_id (c->buffer->info[skippy_iter.idx]); unsigned int this_lig_comp = get_lig_comp (c->buffer->info[skippy_iter.idx]); @@ -982,99 +989,81 @@ static inline void recurse_lookups (context_t *c, static inline bool apply_lookup (hb_apply_context_t *c, unsigned int count, /* Including the first glyph */ - const USHORT input[], /* Array of input values--start with second glyph */ - match_func_t match_func, - const void *match_data, + unsigned int match_positions[MAX_CONTEXT_LENGTH], /* Including the first glyph */ unsigned int lookupCount, - const LookupRecord lookupRecord[] /* Array of LookupRecords--in design order */) + const LookupRecord lookupRecord[], /* Array of LookupRecords--in design order */ + unsigned int match_length) { TRACE_APPLY (NULL); - unsigned int end = c->buffer->len; - if (unlikely (count == 0 || c->buffer->idx + count > end)) - return TRACE_RETURN (false); - - /* TODO We don't support lookupRecord arrays that are not increasing: - * Should be easy for in_place ones at least. */ + hb_buffer_t *buffer = c->buffer; + unsigned int end; - /* Note: If sublookup is reverse, it will underflow after the first loop - * and we jump out of it. Not entirely disastrous. So we don't check - * for reverse lookup here. - */ + /* All positions are distance from beginning of *output* buffer. + * Adjust. */ + { + unsigned int bl = buffer->backtrack_len (); + end = bl + match_length; - hb_apply_context_t::skipping_forward_iterator_t skippy_iter (c, c->buffer->idx, count - 1); - skippy_iter.set_match_func (match_func, match_data, input); - uint8_t syllable = c->buffer->cur().syllable(); + int delta = bl - buffer->idx; + /* Convert positions to new indexing. */ + for (unsigned int j = 0; j < count; j++) + match_positions[j] += delta; + } - unsigned int i = 0; - if (lookupCount && 0 == lookupRecord->sequenceIndex) + for (unsigned int i = 0; i < lookupCount; i++) { - unsigned int old_pos = c->buffer->idx; + unsigned int idx = lookupRecord[i].sequenceIndex; + if (idx >= count) + continue; - /* Apply a lookup */ - bool done = c->recurse (lookupRecord->lookupListIndex); + buffer->move_to (match_positions[idx]); - lookupRecord++; - lookupCount--; - i++; + unsigned int orig_len = buffer->backtrack_len () + buffer->lookahead_len (); + if (!c->recurse (lookupRecord[i].lookupListIndex)) + continue; - if (!done) - goto not_applied; - else - { - if (c->table_index == 1) - c->buffer->idx = old_pos + 1; - /* Reinitialize iterator. */ - hb_apply_context_t::skipping_forward_iterator_t tmp (c, c->buffer->idx - 1, count - i); - tmp.set_syllable (syllable); - skippy_iter = tmp; - } - } - else - { - not_applied: - /* No lookup applied for this index */ - c->buffer->next_glyph (); - i++; - } - while (i < count) - { - if (!skippy_iter.next ()) return TRACE_RETURN (true); - while (c->buffer->idx < skippy_iter.idx) - c->buffer->next_glyph (); + unsigned int new_len = buffer->backtrack_len () + buffer->lookahead_len (); + int delta = new_len - orig_len; - if (lookupCount && i == lookupRecord->sequenceIndex) - { - unsigned int old_pos = c->buffer->idx; + if (!delta) + continue; - /* Apply a lookup */ - bool done = c->recurse (lookupRecord->lookupListIndex); + /* Recursed lookup changed buffer len. Adjust. */ - lookupRecord++; - lookupCount--; - i++; + end += delta; - if (!done) - goto not_applied2; - else - { - if (c->table_index == 1) - c->buffer->idx = old_pos + 1; - /* Reinitialize iterator. */ - hb_apply_context_t::skipping_forward_iterator_t tmp (c, c->buffer->idx - 1, count - i); - tmp.set_syllable (syllable); - skippy_iter = tmp; - } + unsigned int next = idx + 1; /* next now is the position after the recursed lookup. */ + + if (delta > 0) + { + if (unlikely (delta + count > MAX_CONTEXT_LENGTH)) + break; } else { - not_applied2: - /* No lookup applied for this index */ - c->buffer->next_glyph (); - i++; + /* NOTE: delta is negative. */ + delta = MAX (delta, (int) next - (int) count); + next -= delta; } + + /* Shift! */ + memmove (match_positions + next + delta, match_positions + next, + (count - next) * sizeof (match_positions[0])); + next += delta; + count += delta; + + /* Fill in new entries. */ + for (unsigned int j = idx + 1; j < next; j++) + match_positions[j] = match_positions[j - 1] + 1; + + /* And fixup the rest. */ + for (; next < count; next++) + match_positions[next] += delta; } + buffer->move_to (end); + return TRACE_RETURN (true); } @@ -1146,13 +1135,16 @@ static inline bool context_apply_lookup (hb_apply_context_t *c, const LookupRecord lookupRecord[], ContextApplyLookupContext &lookup_context) { + unsigned int match_length = 0; + unsigned int match_positions[MAX_CONTEXT_LENGTH]; return match_input (c, inputCount, input, - lookup_context.funcs.match, lookup_context.match_data) + lookup_context.funcs.match, lookup_context.match_data, + &match_length, match_positions) && apply_lookup (c, - inputCount, input, - lookup_context.funcs.match, lookup_context.match_data, - lookupCount, lookupRecord); + inputCount, match_positions, + lookupCount, lookupRecord, + match_length); } struct Rule @@ -1726,22 +1718,23 @@ static inline bool chain_context_apply_lookup (hb_apply_context_t *c, const LookupRecord lookupRecord[], ChainContextApplyLookupContext &lookup_context) { - unsigned int lookahead_offset = 0; + unsigned int match_length = 0; + unsigned int match_positions[MAX_CONTEXT_LENGTH]; return match_input (c, inputCount, input, lookup_context.funcs.match, lookup_context.match_data[1], - &lookahead_offset) + &match_length, match_positions) && match_backtrack (c, backtrackCount, backtrack, lookup_context.funcs.match, lookup_context.match_data[0]) && match_lookahead (c, lookaheadCount, lookahead, lookup_context.funcs.match, lookup_context.match_data[2], - lookahead_offset) + match_length) && apply_lookup (c, - inputCount, input, - lookup_context.funcs.match, lookup_context.match_data[1], - lookupCount, lookupRecord); + inputCount, match_positions, + lookupCount, lookupRecord, + match_length); } struct ChainRule commit 841e20d083aec8d814cd8d90aa6ab60127c0d1f2 Author: Behdad Esfahbod <[email protected]> Date: Mon Oct 14 18:47:51 2013 +0200 Add test suite for shaping results The new test suite runs tests included under hb/test/shaping/tests/*.tests, which themselves reference font files stored by sha1sum under hb/test/shaping/fonts/sha1sum. The fonts are produced using a subsetter to only include glyphs needed to run the test. Four initial tests are added for (Chain)Context matching, of which three currently fail. diff --git a/test/shaping/Makefile.am b/test/shaping/Makefile.am index 4fb762c..2395783 100644 --- a/test/shaping/Makefile.am +++ b/test/shaping/Makefile.am @@ -7,7 +7,7 @@ DISTCLEANFILES = MAINTAINERCLEANFILES = manifests: - @$(srcdir)/hb-manifest-update "$(srcdir)/texts" "$(srcdir)/fonts" + @$(srcdir)/hb-manifest-update "$(srcdir)/texts" "$(srcdir)/fonts" "$(srcdir)/tests" EXTRA_DIST += \ hb-diff \ @@ -20,6 +20,7 @@ EXTRA_DIST += \ hb-unicode-decode \ hb-unicode-encode \ hb-unicode-prettyname \ + run-tests.sh \ $(NULL) # TODO Figure out Python stuff @@ -30,6 +31,13 @@ CLEANFILES += \ hb_test_tools.py[co] \ $(NULL) +TESTS = $(srcdir)/tests/*.tests +TESTS_ENVIRONMENT = \ + srcdir="$(srcdir)" \ + builddir="$(builddir)" \ + $(srcdir)/run-tests.sh \ + $(NULL) + .PHONY: manifests -include $(top_srcdir)/git.mk diff --git a/test/shaping/fonts/sha1sum/4cce528e99f600ed9c25a2b69e32eb94a03b4ae8.ttf b/test/shaping/fonts/sha1sum/4cce528e99f600ed9c25a2b69e32eb94a03b4ae8.ttf new file mode 100644 index 0000000..dfaead7 Binary files /dev/null and b/test/shaping/fonts/sha1sum/4cce528e99f600ed9c25a2b69e32eb94a03b4ae8.ttf differ diff --git a/test/shaping/fonts/sha1sum/ceadd106a8205214fbe7337ef9de32a862b59762.ttf b/test/shaping/fonts/sha1sum/ceadd106a8205214fbe7337ef9de32a862b59762.ttf new file mode 100644 index 0000000..b37428e Binary files /dev/null and b/test/shaping/fonts/sha1sum/ceadd106a8205214fbe7337ef9de32a862b59762.ttf differ diff --git a/test/shaping/fonts/sha1sum/d629e7fedc0b350222d7987345fe61613fa3929a.ttf b/test/shaping/fonts/sha1sum/d629e7fedc0b350222d7987345fe61613fa3929a.ttf new file mode 100644 index 0000000..e674a78 Binary files /dev/null and b/test/shaping/fonts/sha1sum/d629e7fedc0b350222d7987345fe61613fa3929a.ttf differ diff --git a/test/shaping/fonts/sha1sum/f499fbc23865022234775c43503bba2e63978fe1.ttf b/test/shaping/fonts/sha1sum/f499fbc23865022234775c43503bba2e63978fe1.ttf new file mode 100644 index 0000000..3c60593 Binary files /dev/null and b/test/shaping/fonts/sha1sum/f499fbc23865022234775c43503bba2e63978fe1.ttf differ diff --git a/test/shaping/run-tests.sh b/test/shaping/run-tests.sh new file mode 100755 index 0000000..1da2064 --- /dev/null +++ b/test/shaping/run-tests.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +test "x$srcdir" = x && srcdir=. +test "x$builddir" = x && builddir=. +test "x$top_builddir" = x && top_builddir=../.. + +hb_shape=$top_builddir/util/hb-shape + +fails=0 + +if test $# = 0; then + set /dev/stdin +fi + +IFS=: +for f in "$@"; do + echo "Running tests in $f" + while read fontfile unicodes glyphs_expected; do + echo "Testing $fontfile:$unicodes" + glyphs=`$srcdir/hb-unicode-encode "$unicodes" | $hb_shape "$srcdir/$fontfile"` + if ! test "x$glyphs" = "x$glyphs_expected"; then + echo "Actual: $glyphs" >&2 + echo "Expected: $glyphs_expected" >&2 + let fails=$fails+1 + fi + done < "$f" +done + +if test $fails != 0; then + echo "$fails tests failed." + exit 1 +else + echo "All tests passed." +fi diff --git a/test/shaping/tests/MANIFEST b/test/shaping/tests/MANIFEST new file mode 100644 index 0000000..8a6157a --- /dev/null +++ b/test/shaping/tests/MANIFEST @@ -0,0 +1 @@ +context-matching.tests diff --git a/test/shaping/tests/context-matching.tests b/test/shaping/tests/context-matching.tests new file mode 100644 index 0000000..0d1d8a0 --- /dev/null +++ b/test/shaping/tests/context-matching.tests @@ -0,0 +1,4 @@ +fonts/sha1sum/4cce528e99f600ed9c25a2b69e32eb94a03b4ae8.ttf:U+1A48,U+1A58,U+1A25,U+1A48,U+1A58,U+1A25,U+1A6E,U+1A63:[uni1A48=0+1212|uni1A25=0+1912|uni1A58=0+0|uni1A48=3+1212|uni1A6E=3+1212|uni1A25=3+1912|uni1A58=3+0|uni1A63=3+1212] +fonts/sha1sum/d629e7fedc0b350222d7987345fe61613fa3929a.ttf:U+0915,U+093F,U+0915,U+093F:[ivowelsign03deva=0+530|kadeva=0+1561|ivowelsign03deva=2+530|kadeva=2+1561] +fonts/sha1sum/f499fbc23865022234775c43503bba2e63978fe1.ttf:U+09B0,U+09CD,U+09A5,U+09CD,U+09AF,U+09C0:[gid1=0+1320|gid13=0+523|gid18=0+545] +fonts/sha1sum/ceadd106a8205214fbe7337ef9de32a862b59762.ttf:U+1014,U+1039,U+1011,U+1014,U+1039,U+1011,U+1014,U+1039,U+1011:[gid4=0+1118|gid5=0@97,0+600|gid4=3+1118|gid5=3@97,0+600|gid4=6+1118|gid5=6@97,0+0] commit e2dab69291a5d86fc90a8c273c458c16574eafb5 Author: Behdad Esfahbod <[email protected]> Date: Mon Oct 14 16:44:44 2013 +0200 Minor diff --git a/test/shaping/hb_test_tools.py b/test/shaping/hb_test_tools.py index ccb0e1c..7674fdf 100644 --- a/test/shaping/hb_test_tools.py +++ b/test/shaping/hb_test_tools.py @@ -405,7 +405,7 @@ class Unicode: @staticmethod def decode (s): - return '<' + u','.join ("U+%04X" % ord (u) for u in unicode (s, 'utf-8')).encode ('utf-8') + '>' + return u','.join ("U+%04X" % ord (u) for u in unicode (s, 'utf-8')).encode ('utf-8') @staticmethod def parse (s): commit 4e6e53db5da0a5da87ae732c3f9d01babf4ae6c2 Author: Behdad Esfahbod <[email protected]> Date: Mon Oct 14 13:06:36 2013 +0200 [otlayout] "Minor" diff --git a/src/hb-ot-layout.cc b/src/hb-ot-layout.cc index 07c093f..5f3eb57 100644 --- a/src/hb-ot-layout.cc +++ b/src/hb-ot-layout.cc @@ -429,6 +429,7 @@ hb_ot_layout_table_get_lookup_count (hb_face_t *face, return hb_ot_layout_from_face (face)->gpos_lookup_count; } } + return 0; } static void _______________________________________________ HarfBuzz mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/harfbuzz
