airborne12 commented on code in PR #67918:
URL: https://github.com/apache/doris/pull/67918#discussion_r4069260081
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -226,26 +681,321 @@ private static String resolveTokenFilterIdentity(String
filterList) {
* IMPORTANT: Order is preserved because filter order is semantically
significant.
*/
private static String resolveCharFilterIdentity(String filterList) {
+ return resolveCharFilterIdentity(filterList, false);
+ }
+
+ private static String resolveCharFilterIdentity(String filterList, boolean
lowercaseDownstream) {
+ ArrayDeque<String> identities = new ArrayDeque<>();
+ walkCharFilters(filterList, lowercaseDownstream, identities);
+ return String.join(",", identities);
+ }
+
+ /**
+ * Resolve the chain from its last filter to its first, collecting
identities, and return the
+ * case-folding context that a filter placed in front of the chain would
run in.
+ */
+ private static boolean[] walkCharFilters(
+ String filterList, boolean lowercaseDownstream, Deque<String>
identities) {
+ boolean[] foldBlockedBytes = lowercaseDownstream ? new boolean[256] :
null;
if (Strings.isNullOrEmpty(filterList)) {
- return "";
+ return foldBlockedBytes;
}
- StringBuilder sb = new StringBuilder();
String[] filters = filterList.split(",\\s*");
// DO NOT sort - filter order is semantically significant
- for (int i = 0; i < filters.length; i++) {
- String filter = filters[i].trim();
- if (i > 0) {
- sb.append(",");
+ for (int i = filters.length - 1; i >= 0; --i) {
+ String filterName = filters[i].trim();
+ String filter = resolveComponentIdentity(
+ filterName, IndexPolicyTypeEnum.CHAR_FILTER,
foldBlockedBytes);
+ if (Strings.isNullOrEmpty(filter)) {
+ continue;
}
+ identities.addFirst(filter);
+ foldBlockedBytes = foldBlockedBytesBefore(filterName,
foldBlockedBytes);
+ }
+ return foldBlockedBytes;
+ }
- if (IndexPolicy.BUILTIN_CHAR_FILTERS.contains(filter)) {
- sb.append(filter);
- } else {
- sb.append(resolveComponentIdentity(filter,
IndexPolicyTypeEnum.CHAR_FILTER));
+ /**
+ * Context for the filter that runs before this one: a case fold starts a
fresh context, a
+ * char_replace filter adds the bytes it rewrites, and any other filter
ends the context.
+ */
+ private static boolean[] foldBlockedBytesBefore(String filterName,
boolean[] foldBlockedBytes) {
+ if (isCaseFoldingCharFilter(filterName)) {
+ return new boolean[256];
+ }
+ if (foldBlockedBytes == null) {
+ return null;
+ }
+ boolean[] sourceBytes = charReplaceSourceBytes(filterName);
+ if (sourceBytes == null) {
+ return null;
+ }
+ for (int i = 0; i < foldBlockedBytes.length; ++i) {
+ foldBlockedBytes[i] |= sourceBytes[i];
+ }
+ return foldBlockedBytes;
+ }
+
+ /** Bytes a named char_replace filter rewrites, or null for any other
filter. */
+ private static boolean[] charReplaceSourceBytes(String filterName) {
+ IndexPolicy policy = findPolicy(filterName,
IndexPolicyTypeEnum.CHAR_FILTER);
+ if (policy == null || policy.isInvalid() || policy.getProperties() ==
null) {
+ return null;
+ }
+ Map<String, String> properties = policy.getProperties();
+ String type = normalizeBuiltinComponentName(
+ properties.get(IndexPolicy.PROP_TYPE),
IndexPolicyTypeEnum.CHAR_FILTER);
+ String pattern = properties.get("pattern");
+ if (!"char_replace".equals(type) || pattern == null) {
+ return null;
+ }
+ boolean[] sourceBytes = new boolean[256];
+ for (int i = 0; i < pattern.length(); ++i) {
+ char patternByte = pattern.charAt(i);
+ if (patternByte < sourceBytes.length) {
+ sourceBytes[patternByte] = true;
}
}
- return sb.toString();
+ return sourceBytes;
+ }
+
+ /** The named policy when one exists with the expected type, or null. */
+ private static IndexPolicy findPolicy(String name, IndexPolicyTypeEnum
expectedType) {
+ if (Strings.isNullOrEmpty(name)) {
+ return null;
+ }
+ try {
+ Env env = Env.getCurrentEnv();
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() == expectedType) {
+ return policy;
+ }
+ }
+ } catch (RuntimeException e) {
+ // Treat lookup failures as an unknown policy.
+ }
+ return null;
+ }
+
+ private static boolean isCaseFoldingCharFilter(String name) {
+ if (Strings.isNullOrEmpty(name)) {
+ return false;
+ }
+
+ try {
+ Env env = Env.getCurrentEnv();
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() ==
IndexPolicyTypeEnum.CHAR_FILTER) {
+ if (policy.isInvalid()) {
+ return false;
+ }
+ Map<String, String> properties = policy.getProperties();
+ if (properties != null && !properties.isEmpty()) {
+ String type = normalizeBuiltinComponentName(
+ properties.get(IndexPolicy.PROP_TYPE),
IndexPolicyTypeEnum.CHAR_FILTER);
+ String normalizer = properties.getOrDefault("name",
"nfkc_cf").trim();
+ String unicodeSet =
properties.getOrDefault("unicode_set_filter", "").trim();
+ return "icu_normalizer".equals(type)
+ && "nfkc_cf".equalsIgnoreCase(normalizer)
+ && unicodeSet.isEmpty();
+ }
+ }
+ }
+ } catch (RuntimeException e) {
+ // Fall through to built-in resolution.
+ }
+
+ return "icu_normalizer".equals(
+ normalizeBuiltinComponentName(name,
IndexPolicyTypeEnum.CHAR_FILTER));
+ }
+
+ /** The outer char filter runs before everything else, so it takes the
analyzer's fold context. */
+ private static String appendOuterCharFilterIdentity(
+ String analyzerIdentity, Map<String, String> properties, boolean[]
foldBlockedBytes) {
+ String type =
properties.get(InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE);
+ String pattern =
properties.get(InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN);
+ if (!"char_replace".equals(type) || Strings.isNullOrEmpty(pattern)) {
+ return analyzerIdentity;
+ }
+ String replacement = properties.getOrDefault(
+
InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " ");
+ String canonicalPattern = canonicalizeCharReplacePattern(pattern,
replacement, foldBlockedBytes);
+ if (canonicalPattern.isEmpty()) {
+ return analyzerIdentity;
+ }
+ return analyzerIdentity + "|outer_char_filter=char_replace:"
+ + canonicalPattern.length() + ":" + canonicalPattern + ":"
+ + replacement.length() + ":" + replacement + ";";
+ }
+
+ /**
+ * Canonicalize the ASCII pattern to the BE filter's byte set.
+ * Order, duplicate bytes, and replacements of a byte with itself do not
change the stream.
+ */
+ private static String canonicalizeCharReplacePattern(
+ String pattern, String replacement, boolean[] foldBlockedBytes) {
+ if (replacement.length() != 1) {
+ return pattern;
+ }
+ char replacementByte = replacement.charAt(0);
+ boolean[] replacedBytes = new boolean[256];
+ for (int i = 0; i < pattern.length(); ++i) {
+ char patternByte = pattern.charAt(i);
+ if (patternByte < replacedBytes.length && patternByte !=
replacementByte) {
+ replacedBytes[patternByte] = true;
+ }
+ }
+ if (foldBlockedBytes != null && replacementByte >= 'a' &&
replacementByte <= 'z') {
+ // The downstream fold maps the upper-case byte to the replacement
anyway, unless a
+ // filter in between rewrites either byte.
+ int upperByte = replacementByte - ('a' - 'A');
+ if (!foldBlockedBytes[upperByte] &&
!foldBlockedBytes[replacementByte]) {
+ replacedBytes[upperByte] = false;
+ }
+ }
+
+ StringBuilder canonical = new StringBuilder();
+ for (int i = 0; i < replacedBytes.length; ++i) {
+ if (replacedBytes[i]) {
+ canonical.append((char) i);
+ }
+ }
+ return canonical.toString();
+ }
+
+ private static boolean[] builtinIkFoldContext(String analyzerIdentity) {
+ return isDefaultLowercaseBuiltinIkIdentity(analyzerIdentity) ? new
boolean[256] : null;
+ }
+
+ private static boolean isDefaultLowercaseBuiltinIkIdentity(String
analyzerIdentity) {
+ return (IndexPolicyTypeEnum.ANALYZER.name() +
":tokenizer=ik_smart;").equals(analyzerIdentity)
+ || (IndexPolicyTypeEnum.ANALYZER.name() +
":tokenizer=ik_max_word;").equals(analyzerIdentity);
+ }
+
+ /**
+ * Fold context for the outer char filter of a custom analyzer, which BE
applies before the
+ * analyzer's own char filters. Unknown or unresolvable analyzers get no
context.
Review Comment:
Fixed in 7345ff82676. Confirmed on BE: the column writer builds the outer
`char_replace` reader from the index properties for every column,
`get_analyzer_by_name` returns a `CustomNormalizer` for a NORMALIZER policy,
and its `init_reader` stacks the policy's own char filters on top of that outer
reader with the keyword tokenizer. Reproduced first: three `NORMALIZER:`
identities kept the outer/inner `A -> a` although the pipeline is `keyword ->
lowercase`.
`customAnalyzerFoldContext` now falls back to the NORMALIZER policy when no
ANALYZER of that name exists, and `foldsAsciiCaseAfterCharFilters` takes the
policy type so a normalizer is judged as keyword (case transparent) plus its
token-filter chain; the normalizer's own identity uses the same starting
context. `testOuterCharFilterAbsorbedByNormalizerPipeline` covers the aliases
with and without the outer filter; CREATE is in
`testCreateTableRejectsOuterCaseFoldAbsorbedByNormalizerAliases` and ALTER in
`testAddInvertedIndexRejectsFoldAliasesThroughEmptySetNormalizerAndTransparentFilters`.
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java:
##########
@@ -226,26 +681,321 @@ private static String resolveTokenFilterIdentity(String
filterList) {
* IMPORTANT: Order is preserved because filter order is semantically
significant.
*/
private static String resolveCharFilterIdentity(String filterList) {
+ return resolveCharFilterIdentity(filterList, false);
+ }
+
+ private static String resolveCharFilterIdentity(String filterList, boolean
lowercaseDownstream) {
+ ArrayDeque<String> identities = new ArrayDeque<>();
+ walkCharFilters(filterList, lowercaseDownstream, identities);
+ return String.join(",", identities);
+ }
+
+ /**
+ * Resolve the chain from its last filter to its first, collecting
identities, and return the
+ * case-folding context that a filter placed in front of the chain would
run in.
+ */
+ private static boolean[] walkCharFilters(
+ String filterList, boolean lowercaseDownstream, Deque<String>
identities) {
+ boolean[] foldBlockedBytes = lowercaseDownstream ? new boolean[256] :
null;
if (Strings.isNullOrEmpty(filterList)) {
- return "";
+ return foldBlockedBytes;
}
- StringBuilder sb = new StringBuilder();
String[] filters = filterList.split(",\\s*");
// DO NOT sort - filter order is semantically significant
- for (int i = 0; i < filters.length; i++) {
- String filter = filters[i].trim();
- if (i > 0) {
- sb.append(",");
+ for (int i = filters.length - 1; i >= 0; --i) {
+ String filterName = filters[i].trim();
+ String filter = resolveComponentIdentity(
+ filterName, IndexPolicyTypeEnum.CHAR_FILTER,
foldBlockedBytes);
+ if (Strings.isNullOrEmpty(filter)) {
+ continue;
}
+ identities.addFirst(filter);
+ foldBlockedBytes = foldBlockedBytesBefore(filterName,
foldBlockedBytes);
+ }
+ return foldBlockedBytes;
+ }
- if (IndexPolicy.BUILTIN_CHAR_FILTERS.contains(filter)) {
- sb.append(filter);
- } else {
- sb.append(resolveComponentIdentity(filter,
IndexPolicyTypeEnum.CHAR_FILTER));
+ /**
+ * Context for the filter that runs before this one: a case fold starts a
fresh context, a
+ * char_replace filter adds the bytes it rewrites, and any other filter
ends the context.
+ */
+ private static boolean[] foldBlockedBytesBefore(String filterName,
boolean[] foldBlockedBytes) {
+ if (isCaseFoldingCharFilter(filterName)) {
+ return new boolean[256];
+ }
+ if (foldBlockedBytes == null) {
+ return null;
+ }
+ boolean[] sourceBytes = charReplaceSourceBytes(filterName);
+ if (sourceBytes == null) {
+ return null;
+ }
+ for (int i = 0; i < foldBlockedBytes.length; ++i) {
+ foldBlockedBytes[i] |= sourceBytes[i];
+ }
+ return foldBlockedBytes;
+ }
+
+ /** Bytes a named char_replace filter rewrites, or null for any other
filter. */
+ private static boolean[] charReplaceSourceBytes(String filterName) {
+ IndexPolicy policy = findPolicy(filterName,
IndexPolicyTypeEnum.CHAR_FILTER);
+ if (policy == null || policy.isInvalid() || policy.getProperties() ==
null) {
+ return null;
+ }
+ Map<String, String> properties = policy.getProperties();
+ String type = normalizeBuiltinComponentName(
+ properties.get(IndexPolicy.PROP_TYPE),
IndexPolicyTypeEnum.CHAR_FILTER);
+ String pattern = properties.get("pattern");
+ if (!"char_replace".equals(type) || pattern == null) {
+ return null;
+ }
+ boolean[] sourceBytes = new boolean[256];
+ for (int i = 0; i < pattern.length(); ++i) {
+ char patternByte = pattern.charAt(i);
+ if (patternByte < sourceBytes.length) {
+ sourceBytes[patternByte] = true;
}
}
- return sb.toString();
+ return sourceBytes;
+ }
+
+ /** The named policy when one exists with the expected type, or null. */
+ private static IndexPolicy findPolicy(String name, IndexPolicyTypeEnum
expectedType) {
+ if (Strings.isNullOrEmpty(name)) {
+ return null;
+ }
+ try {
+ Env env = Env.getCurrentEnv();
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() == expectedType) {
+ return policy;
+ }
+ }
+ } catch (RuntimeException e) {
+ // Treat lookup failures as an unknown policy.
+ }
+ return null;
+ }
+
+ private static boolean isCaseFoldingCharFilter(String name) {
+ if (Strings.isNullOrEmpty(name)) {
+ return false;
+ }
+
+ try {
+ Env env = Env.getCurrentEnv();
+ if (env != null && env.getIndexPolicyMgr() != null) {
+ IndexPolicy policy =
env.getIndexPolicyMgr().getPolicyByName(name);
+ if (policy != null && policy.getType() ==
IndexPolicyTypeEnum.CHAR_FILTER) {
+ if (policy.isInvalid()) {
+ return false;
+ }
+ Map<String, String> properties = policy.getProperties();
+ if (properties != null && !properties.isEmpty()) {
+ String type = normalizeBuiltinComponentName(
+ properties.get(IndexPolicy.PROP_TYPE),
IndexPolicyTypeEnum.CHAR_FILTER);
+ String normalizer = properties.getOrDefault("name",
"nfkc_cf").trim();
+ String unicodeSet =
properties.getOrDefault("unicode_set_filter", "").trim();
+ return "icu_normalizer".equals(type)
+ && "nfkc_cf".equalsIgnoreCase(normalizer)
+ && unicodeSet.isEmpty();
+ }
+ }
+ }
+ } catch (RuntimeException e) {
+ // Fall through to built-in resolution.
+ }
+
+ return "icu_normalizer".equals(
+ normalizeBuiltinComponentName(name,
IndexPolicyTypeEnum.CHAR_FILTER));
+ }
+
+ /** The outer char filter runs before everything else, so it takes the
analyzer's fold context. */
+ private static String appendOuterCharFilterIdentity(
+ String analyzerIdentity, Map<String, String> properties, boolean[]
foldBlockedBytes) {
+ String type =
properties.get(InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE);
+ String pattern =
properties.get(InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN);
+ if (!"char_replace".equals(type) || Strings.isNullOrEmpty(pattern)) {
+ return analyzerIdentity;
+ }
+ String replacement = properties.getOrDefault(
+
InvertedIndexProperties.INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT, " ");
+ String canonicalPattern = canonicalizeCharReplacePattern(pattern,
replacement, foldBlockedBytes);
+ if (canonicalPattern.isEmpty()) {
+ return analyzerIdentity;
+ }
+ return analyzerIdentity + "|outer_char_filter=char_replace:"
+ + canonicalPattern.length() + ":" + canonicalPattern + ":"
+ + replacement.length() + ":" + replacement + ";";
+ }
+
+ /**
+ * Canonicalize the ASCII pattern to the BE filter's byte set.
+ * Order, duplicate bytes, and replacements of a byte with itself do not
change the stream.
+ */
+ private static String canonicalizeCharReplacePattern(
+ String pattern, String replacement, boolean[] foldBlockedBytes) {
+ if (replacement.length() != 1) {
+ return pattern;
+ }
+ char replacementByte = replacement.charAt(0);
+ boolean[] replacedBytes = new boolean[256];
+ for (int i = 0; i < pattern.length(); ++i) {
+ char patternByte = pattern.charAt(i);
+ if (patternByte < replacedBytes.length && patternByte !=
replacementByte) {
+ replacedBytes[patternByte] = true;
+ }
+ }
+ if (foldBlockedBytes != null && replacementByte >= 'a' &&
replacementByte <= 'z') {
+ // The downstream fold maps the upper-case byte to the replacement
anyway, unless a
+ // filter in between rewrites either byte.
+ int upperByte = replacementByte - ('a' - 'A');
+ if (!foldBlockedBytes[upperByte] &&
!foldBlockedBytes[replacementByte]) {
+ replacedBytes[upperByte] = false;
+ }
+ }
+
+ StringBuilder canonical = new StringBuilder();
+ for (int i = 0; i < replacedBytes.length; ++i) {
+ if (replacedBytes[i]) {
+ canonical.append((char) i);
+ }
+ }
+ return canonical.toString();
+ }
+
+ private static boolean[] builtinIkFoldContext(String analyzerIdentity) {
+ return isDefaultLowercaseBuiltinIkIdentity(analyzerIdentity) ? new
boolean[256] : null;
+ }
+
+ private static boolean isDefaultLowercaseBuiltinIkIdentity(String
analyzerIdentity) {
+ return (IndexPolicyTypeEnum.ANALYZER.name() +
":tokenizer=ik_smart;").equals(analyzerIdentity)
+ || (IndexPolicyTypeEnum.ANALYZER.name() +
":tokenizer=ik_max_word;").equals(analyzerIdentity);
+ }
+
+ /**
+ * Fold context for the outer char filter of a custom analyzer, which BE
applies before the
+ * analyzer's own char filters. Unknown or unresolvable analyzers get no
context.
+ */
+ private static boolean[] customAnalyzerFoldContext(String analyzerName) {
+ if (IndexPolicy.BUILTIN_ANALYZERS.contains(analyzerName)
+ || IndexPolicy.BUILTIN_NORMALIZERS.contains(analyzerName)) {
+ return null;
+ }
+ IndexPolicy policy = findPolicy(analyzerName,
IndexPolicyTypeEnum.ANALYZER);
+ if (policy == null || policy.isInvalid() || policy.getProperties() ==
null
+ || policy.getProperties().isEmpty()) {
+ return null;
+ }
+ Map<String, String> properties = policy.getProperties();
+ try {
+ String tokenizerIdentity = resolveComponentIdentity(
+ properties.get(IndexPolicy.PROP_TOKENIZER),
IndexPolicyTypeEnum.TOKENIZER);
+ return
walkCharFilters(properties.get(IndexPolicy.PROP_CHAR_FILTER),
+ foldsAsciiCaseAfterCharFilters(properties,
tokenizerIdentity), new ArrayDeque<>());
+ } catch (RuntimeException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Whether the tokenizer and token filters emit the same tokens for an
ASCII letter of either
+ * case, so a char filter that only lowercases such a letter cannot change
the output.
+ */
+ private static boolean foldsAsciiCaseAfterCharFilters(
+ Map<String, String> properties, String tokenizerIdentity) {
+ if ("ik_smart".equals(tokenizerIdentity) ||
"ik_max_word".equals(tokenizerIdentity)) {
+ return true;
Review Comment:
Fixed in 7345ff82676. Confirmed on BE: `ASCIIFoldingFilter` only folds code
points >= U+0080 and copies ASCII bytes unchanged (including with
`preserve_original`, which adds no duplicate for pure ASCII), and
`LowerCaseFilter` lowercases the full Unicode range, so `keyword ->
asciifolding -> lowercase` absorbs an outer `A -> a`. Reproduced first:
`asciifolding,lowercase`, `asciifolding` with `preserve_original`, `nfc ->
lowercase`, a default `icu_normalizer` token filter, and `asciifolding ->
icu_normalizer([])` all kept the outer suffix.
The first-filter check is replaced by `tokenFiltersFoldAsciiCase`, an
in-order scan: `lowercase` or an unfiltered default `nfkc_cf` `icu_normalizer`
proves the fold, `asciifolding` and `nfc`/`nfd`/`nfkc`/`nfkd` normalizers are
ASCII-case transparent and are skipped, and anything else (word_delimiter,
pinyin, a filtered `nfkc_cf`, unknown policies) stops the scan.
`testOuterCharFilterAbsorbedThroughAsciiTransparentTokenFilters` covers those
five orderings, and
`testCreateTableRejectsOuterCaseFoldThroughAsciiTransparentFilters` covers
CREATE with a `word_delimiter,lowercase` negative; ALTER is in the shared case
from the previous thread.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
