Gabriel39 commented on code in PR #69672:
URL: https://github.com/apache/doris/pull/69672#discussion_r4409484054
##########
fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsMappingUtils.java:
##########
@@ -150,12 +190,155 @@ public static EsFieldContext
resolveFieldContext(List<String> columnNames,
String fieldType = fieldObject.get("type").asText();
fieldContext.getColumn2typeMap().put(colName, fieldType);
resolveDateFields(fieldContext, fieldObject, colName, fieldType);
- resolveKeywordFields(fieldContext, fieldObject, colName,
fieldType);
resolveDocValuesFields(fieldContext, fieldObject, colName,
fieldType);
}
+ resolveSourceValueSafety(fieldContext, indexMapping, mappingType,
properties);
return fieldContext;
}
+ private static void resolveSourceValueSafety(EsFieldContext context,
String indexMapping, String mappingType,
+ ObjectNode schemaProperties) {
+ try {
+ // Mapping-induced source/index differences on any alias backing
index must prevent
+ // filtering or replacing source values through that path.
+ List<ObjectNode> roots = new ArrayList<>();
+ Set<String> columns = new
HashSet<>(context.getColumn2typeMap().keySet());
+ for (JsonNode index : MAPPER.readTree(indexMapping)) {
+ ObjectNode root = getRootSchema((ObjectNode)
index.get("mappings"), mappingType,
+ new ArrayList<>(), true);
+ roots.add(root);
+ }
+ // Use the same populated schema as inference, including
unprojected filter columns.
+ schemaProperties.fieldNames().forEachRemaining(columns::add);
+ // Filters can reference columns absent from the projection.
Resolve their query paths
+ // from the exposed schema too; the projection still controls
docvalue read eligibility.
+ for (String column : columns) {
+ ObjectNode field = (ObjectNode) schemaProperties.get(column);
+ resolveKeywordFields(context, field, column,
field.path("type").asText());
+ }
+ Set<String> unsafeDocValues = new HashSet<>();
+ for (ObjectNode root : roots) {
+ JsonNode properties = root.path("properties");
+ if (isSourceDisabled(root.path("_source"))) {
+ // A source projection fails on disabled backings; full
source preserves the
+ // enabled backings of a mixed alias while disabled hits
remain NULL.
+ context.requireFullSourceRead();
+ }
+ Set<String> copyTargets = new HashSet<>();
+ collectCopyTargets(properties, copyTargets);
+ for (String column : columns) {
+ JsonNode field = properties.path(column);
+ JsonNode queryField = fieldAtPath(field, column,
context.getFetchFieldsContext().get(column));
+ JsonNode readField = fieldAtPath(field, column,
context.getDocValueFieldsContext().get(column));
+ // Disabled root parsing leaves source fields but produces
no index/docvalues,
+ // even if an explicit properties section still describes
those fields.
+ boolean sourceChanged =
!root.path("enabled").asBoolean(true)
+ || filtersSourceColumn(root.path("_source"),
column)
+ || root.path("runtime").has(column) ||
copyTargets.contains(column);
+ // ES mappings do not distinguish arrays from scalars.
Even a short equality can
+ // match one indexed array element while Doris reads the
entire source JSON string.
+ if (sourceChanged || changesSourceValue(field) ||
changesSourceValue(queryField)
+ || "text".equals(queryField.path("type").asText())
|| !isSearchable(queryField)) {
+ context.getSourceValueUnsafeFields().add(column);
+ }
+ // Source can omit generated/aliased values, retain
unrounded numbers or decode
+ // default dates in local time. Docvalues remain eligible
for these mappings.
+ if (requiresSourcePredicateEvaluation(field) ||
requiresSourcePredicateEvaluation(queryField)) {
+ context.getSourceModeUnsafeFields().add(column);
+ }
+ // Query and docvalue paths may select different
multi-fields. An unused sibling's
+ // normalizer must not invalidate either path; only the
chosen read value matters.
+ if (sourceChanged || changesSourceValue(field) ||
changesSourceValue(readField)
+ || !readField.path("doc_values").asBoolean(true)
+ ||
DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(readField.path("type").asText())) {
+ // One backing without this docvalue path would return
missing values for its hits.
+ unsafeDocValues.add(column);
+ }
+ }
+ }
+
unsafeDocValues.forEach(context.getDocValueFieldsContext()::remove);
+ } catch (IOException e) {
+ throw new DorisConnectorException("Failed to parse ES mapping: " +
e.getMessage());
+ }
+ }
+
+ private static JsonNode fieldAtPath(JsonNode field, String column, String
path) {
+ return path == null || column.equals(path) ? field
+ : field.path("fields").path(path.substring(column.length() +
1));
+ }
+
+ private static boolean changesSourceValue(JsonNode field) {
+ return field.isMissingNode() || field.has("ignore_above") ||
field.hasNonNull("normalizer")
+ || field.hasNonNull("null_value") ||
field.path("ignore_malformed").asBoolean(false);
Review Comment:
Fixed in c11be7b2769ebf43424deade2b5df79c8a1a30f8. The existing mapping
metadata request now fetches mappings and index.mapping defaults together
through the filtered Get Index API. Safety analysis uses effective
ignore_malformed for supported field types, honors explicit per-field
overrides, and checks every alias backing. Added a request-shape/count test,
mapping and residual-plan tests, and live inherited
coerce=false/ignore_malformed=true cases with field-override and mixed-alias
controls. IS NULL/IS NOT NULL/equality results match an independent internal
table in all eight reader/docvalue/parallel combinations.
##########
fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java:
##########
@@ -206,15 +212,17 @@ private ScanNodePropertiesResult buildScanNodeProperties(
// accepts any casing. Passing the raw string through let "TRUE" mean
https to FE and http to BE.
nodeProps.put(PROP_HTTP_SSL_ENABLED,
String.valueOf(props.isHttpSslEnabled()));
- // Query DSL (with not-pushed conjunct tracking)
- EsQueryDslResult dslResult = buildQueryDsl(filter, state);
- nodeProps.put(PROP_QUERY_DSL, dslResult.getQueryDsl());
+ // The read mode is scan-wide: losing a's docvalue path also changes a
keyword sibling b
+ // from scalar docvalues to source JSON (e.g. ["cat"]). Decide it
before pruning b's predicate.
+ int docValuesMode = useDocValueScan(columns, state);
+ nodeProps.put(PROP_DOC_VALUES_MODE, String.valueOf(docValuesMode));
Review Comment:
This mixed-generation scheduling issue remains a known limitation. The
maintainer explicitly excluded upgrade-compatibility work from the requested
scope, so no backend-generation gate is added. The PR description now states
that source plans depending on the updated decoder require the updated BE.
Resolving this thread as an explicit scope decision; this is not claimed as a
code fix or as smooth-upgrade support.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
