thomasmueller commented on PR #1071:
URL: https://github.com/apache/jackrabbit-oak/pull/1071#issuecomment-1680121413
I would make the truncation method public and then write a good unit test
for it.
private static final Logger log =
LoggerFactory.getLogger(LuceneDocumentMaker.class);
@Test
public void test() {
Random r = new Random(1);
for (int i = 0; i < 1000; i++) {
String x = randomUnicodeString(r, 5);
BytesRef ref = checkTruncateLength("x", x, "/x", 5);
assertTrue(ref.length > 0 && ref.length <= 5);
}
}
private String randomUnicodeString(Random r, int len) {
StringBuilder buff = new StringBuilder();
for(int i=0; i<len; i++) {
// see https://en.wikipedia.org/wiki/UTF-8
switch (r.nextInt(6)) {
case 2:
// 2 UTF-8 bytes
buff.append('£');
break;
case 3:
// 3 UTF-8 bytes
buff.append('€');
break;
case 4:
// 4 UTF-8 bytes
buff.append("\uD800\uDF48");
break;
default:
// most cases:
// 1 UTF-8 byte (ASCII)
buff.append('$');
}
}
return buff.toString();
}
public static BytesRef checkTruncateLength(String prop, String value,
String path, int maxLength) {
BytesRef ref = new BytesRef(value);
if (ref.length <= maxLength) {
return ref;
}
log.info("Truncating property {} at path:[{}] as length after
encoding {} is > {} ",
prop, path, ref.length, maxLength);
int end = maxLength - 1;
// skip over tails of utf-8 multi-byte sequences (up to 3 bytes)
while ((ref.bytes[end] & 0b11000000) == 0b10000000) {
end--;
}
// remove one head of a utf-8 multi-byte sequence (at most 1)
if ((ref.bytes[end] & 0b11000000) == 0b11000000) {
end--;
}
byte[] bytes2 = Arrays.copyOf(ref.bytes, end + 1);
String truncated = new String(bytes2, StandardCharsets.UTF_8);
ref = new BytesRef(truncated);
while (ref.length > maxLength) {
log.error("Truncation did not work: still {} bytes", ref.length);
// this may not properly work with unicode surrogates:
// it is an "emergency" procedure and should never happen
truncated = truncated.substring(0, truncated.length() - 10);
ref = new BytesRef(truncated);
}
return ref;
}
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
