github-actions[bot] commented on code in PR #69672:
URL: https://github.com/apache/doris/pull/69672#discussion_r4398333471
##########
fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java:
##########
@@ -251,26 +254,57 @@ private void serializeFieldContexts(EsMetadataState
state, Map<String, String> n
}
private EsQueryDslResult buildQueryDsl(Optional<ConnectorExpression>
filter,
- EsMetadataState state) {
+ EsMetadataState state, int docValuesMode) {
if (!filter.isPresent()) {
return new EsQueryDslResult("{\"match_all\":{}}",
Collections.emptyList());
}
Map<String, String> fieldsContext = Collections.emptyMap();
List<String> needCompatDateFields = Collections.emptyList();
Map<String, String> column2typeMap = Collections.emptyMap();
+ Set<String> unsafeFields = new HashSet<>();
if (state.getFieldContext() != null) {
if (props.isKeywordSniff()) {
fieldsContext =
state.getFieldContext().getFetchFieldsContext();
}
needCompatDateFields =
state.getFieldContext().getNeedCompatDateFields();
column2typeMap = state.getFieldContext().getColumn2typeMap();
+
unsafeFields.addAll(state.getFieldContext().getSourceValueUnsafeFields());
+ if (docValuesMode == 0) {
+
unsafeFields.addAll(state.getFieldContext().getSourceModeUnsafeFields());
+ }
+ if (!props.isKeywordSniff()) {
+ // Without keyword sniff a term targets analyzed text, even
when a safe raw subfield
+ // exists. Do not apply the unused raw path's safety to the
actual text query.
+
unsafeFields.addAll(state.getFieldContext().getFetchFieldsContext().keySet());
+ }
+ }
+
+ if (docValuesMode == 0) {
+ collectSourceStringFields(filter.get(), unsafeFields);
}
return EsQueryDslBuilder.buildQueryDslWithResult(
filter.get(), fieldsContext, column2typeMap,
- props.isLikePushDown(), needCompatDateFields);
+ props.isLikePushDown(), needCompatDateFields, unsafeFields);
+ }
+
+ private static void collectSourceStringFields(ConnectorExpression
expression, Set<String> fields) {
+ if (expression instanceof ConnectorColumnRef) {
+ ConnectorColumnRef column = (ConnectorColumnRef) expression;
+ String type = column.getType().getTypeName();
+ // Source arrays are serialized as STRING, whereas ES predicates
match their elements.
+ // _id comes from hit metadata and cannot acquire an array
representation in source mode.
+ if (!"_id".equals(column.getColumnName())
+ && ("STRING".equalsIgnoreCase(type) ||
"VARCHAR".equalsIgnoreCase(type)
+ || "CHAR".equalsIgnoreCase(type))) {
Review Comment:
[P2] Retain source-mode nullness predicates on declared ARRAY columns
The type check omits declared ARRAY columns. With
`_meta:{"doris":{"array_fields":["b"]}}`, map `a` as a normalized keyword and
`b` as `long`, then insert `{"a":"ok","b":[]}`. Projecting `a,b` newly forces
source reads, but `b` is ARRAY<BIGINT>, so `SELECT a,b FROM idx WHERE b IS
NULL` still pushes `must_not exists(b)` and drops its residual. ES [treats an
empty array as a missing indexed
field](https://www.elastic.co/guide/en/elasticsearch/reference/8.19/array.html),
while `ScrollParser` inserts a nonnull empty ARRAY from source. The query
consequently returns `(ok,[])` despite `b IS NULL`; `IS NOT NULL` loses that
row. Before this change missing docvalues produced SQL NULL. This concerns
explicit ARRAY container nullness, distinct from the existing inferred-STRING
array equality thread. Include ARRAY references in final source-mode predicate
safety and add empty/nonempty/null typed-array controls across both readers.
##########
fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsMappingUtils.java:
##########
@@ -150,12 +169,139 @@ public static EsFieldContext
resolveFieldContext(List<String> columnNames,
String fieldType = fieldObject.get("type").asText();
fieldContext.getColumn2typeMap().put(colName, fieldType);
resolveDateFields(fieldContext, fieldObject, colName, fieldType);
- resolveKeywordFields(fieldContext, fieldObject, colName,
fieldType);
resolveDocValuesFields(fieldContext, fieldObject, colName,
fieldType);
}
+ resolveSourceValueSafety(fieldContext, indexMapping, mappingType,
properties);
return fieldContext;
}
+ private static void resolveSourceValueSafety(EsFieldContext context,
String indexMapping, String mappingType,
+ ObjectNode schemaProperties) {
+ try {
+ // Mapping-induced source/index differences on any alias backing
index must prevent
+ // filtering or replacing source values through that path.
+ List<ObjectNode> roots = new ArrayList<>();
+ Set<String> columns = new
HashSet<>(context.getColumn2typeMap().keySet());
+ for (JsonNode index : MAPPER.readTree(indexMapping)) {
+ ObjectNode root = getRootSchema((ObjectNode)
index.get("mappings"), mappingType,
+ new ArrayList<>(), true);
+ roots.add(root);
+ }
+ // Use the same populated schema as inference, including
unprojected filter columns.
+ schemaProperties.fieldNames().forEachRemaining(columns::add);
+ // Filters can reference columns absent from the projection.
Resolve their query paths
+ // from the exposed schema too; the projection still controls
docvalue read eligibility.
+ for (String column : columns) {
+ ObjectNode field = (ObjectNode) schemaProperties.get(column);
+ resolveKeywordFields(context, field, column,
field.path("type").asText());
+ }
+ Set<String> unsafeDocValues = new HashSet<>();
+ for (ObjectNode root : roots) {
+ JsonNode properties = root.path("properties");
+ Set<String> copyTargets = new HashSet<>();
+ collectCopyTargets(properties, copyTargets);
+ for (String column : columns) {
+ JsonNode field = properties.path(column);
+ JsonNode queryField = fieldAtPath(field, column,
context.getFetchFieldsContext().get(column));
+ JsonNode readField = fieldAtPath(field, column,
context.getDocValueFieldsContext().get(column));
+ boolean sourceChanged =
filtersSourceColumn(root.path("_source"), column)
+ || root.path("runtime").has(column) ||
copyTargets.contains(column);
+ // ES mappings do not distinguish arrays from scalars.
Even a short equality can
+ // match one indexed array element while Doris reads the
entire source JSON string.
+ if (sourceChanged || changesSourceValue(field) ||
changesSourceValue(queryField)
+ || "text".equals(queryField.path("type").asText())
|| !isSearchable(queryField)) {
+ context.getSourceValueUnsafeFields().add(column);
+ }
+ // These mappings round indexed numbers while _source
retains the original value.
+ // Keep docvalues eligible: only a final source read makes
their predicates unsafe.
+ if (roundsNumericSourceValue(field) ||
roundsNumericSourceValue(queryField)) {
Review Comment:
[P2] Guard fields absent from source after a scan-wide fallback
With docvalues enabled, map `a` as a keyword with a lowercase normalizer,
`n` as `long`, and `b` as `{"type":"alias","path":"n"}`, then insert
`{"a":"ok","n":5}`. `SELECT a,b FROM idx WHERE b=5` previously returned
`(ok,5)` from docvalues. Removing `a`'s docvalue path now forces source for
every column, but `b` is absent from both unsafe sets, so its term query stays
pushed and its SQL residual disappears. Field aliases [support
queries/docvalue_fields but never exist in
_source](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/field-alias);
both readers consequently return `(ok,NULL)`, violating the predicate.
The same gap affects an index-time [scripted numeric
field](https://www.elastic.co/guide/en/elasticsearch/reference/8.19/number.html),
e.g. `b:{"type":"long","script":{"source":"emit(5)"}}` with source
`{"a":"ok"}`: docvalues contain 5 but source has no `b`. This is distinct from
rounding and index-alias routing. Account for both aliases and
mapping-generated fields in source-mode safety, preserving usable docvalues,
and add fallback/alias/script controls across both readers.
##########
fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsMappingUtils.java:
##########
@@ -150,12 +169,139 @@ public static EsFieldContext
resolveFieldContext(List<String> columnNames,
String fieldType = fieldObject.get("type").asText();
fieldContext.getColumn2typeMap().put(colName, fieldType);
resolveDateFields(fieldContext, fieldObject, colName, fieldType);
- resolveKeywordFields(fieldContext, fieldObject, colName,
fieldType);
resolveDocValuesFields(fieldContext, fieldObject, colName,
fieldType);
}
+ resolveSourceValueSafety(fieldContext, indexMapping, mappingType,
properties);
return fieldContext;
}
+ private static void resolveSourceValueSafety(EsFieldContext context,
String indexMapping, String mappingType,
+ ObjectNode schemaProperties) {
+ try {
+ // Mapping-induced source/index differences on any alias backing
index must prevent
+ // filtering or replacing source values through that path.
+ List<ObjectNode> roots = new ArrayList<>();
+ Set<String> columns = new
HashSet<>(context.getColumn2typeMap().keySet());
+ for (JsonNode index : MAPPER.readTree(indexMapping)) {
+ ObjectNode root = getRootSchema((ObjectNode)
index.get("mappings"), mappingType,
+ new ArrayList<>(), true);
+ roots.add(root);
+ }
+ // Use the same populated schema as inference, including
unprojected filter columns.
+ schemaProperties.fieldNames().forEachRemaining(columns::add);
+ // Filters can reference columns absent from the projection.
Resolve their query paths
+ // from the exposed schema too; the projection still controls
docvalue read eligibility.
+ for (String column : columns) {
+ ObjectNode field = (ObjectNode) schemaProperties.get(column);
+ resolveKeywordFields(context, field, column,
field.path("type").asText());
+ }
+ Set<String> unsafeDocValues = new HashSet<>();
+ for (ObjectNode root : roots) {
+ JsonNode properties = root.path("properties");
+ Set<String> copyTargets = new HashSet<>();
+ collectCopyTargets(properties, copyTargets);
+ for (String column : columns) {
+ JsonNode field = properties.path(column);
+ JsonNode queryField = fieldAtPath(field, column,
context.getFetchFieldsContext().get(column));
+ JsonNode readField = fieldAtPath(field, column,
context.getDocValueFieldsContext().get(column));
+ boolean sourceChanged =
filtersSourceColumn(root.path("_source"), column)
+ || root.path("runtime").has(column) ||
copyTargets.contains(column);
+ // ES mappings do not distinguish arrays from scalars.
Even a short equality can
+ // match one indexed array element while Doris reads the
entire source JSON string.
+ if (sourceChanged || changesSourceValue(field) ||
changesSourceValue(queryField)
+ || "text".equals(queryField.path("type").asText())
|| !isSearchable(queryField)) {
+ context.getSourceValueUnsafeFields().add(column);
+ }
+ // These mappings round indexed numbers while _source
retains the original value.
+ // Keep docvalues eligible: only a final source read makes
their predicates unsafe.
+ if (roundsNumericSourceValue(field) ||
roundsNumericSourceValue(queryField)) {
+ context.getSourceModeUnsafeFields().add(column);
+ }
+ // Query and docvalue paths may select different
multi-fields. An unused sibling's
+ // normalizer must not invalidate either path; only the
chosen read value matters.
+ if (sourceChanged || changesSourceValue(field) ||
changesSourceValue(readField)
+ || !readField.path("doc_values").asBoolean(true)
+ ||
DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(readField.path("type").asText())) {
+ // One backing without this docvalue path would return
missing values for its hits.
+ unsafeDocValues.add(column);
+ }
+ }
+ }
+
unsafeDocValues.forEach(context.getDocValueFieldsContext()::remove);
+ } catch (IOException e) {
+ throw new DorisConnectorException("Failed to parse ES mapping: " +
e.getMessage());
+ }
+ }
+
+ private static JsonNode fieldAtPath(JsonNode field, String column, String
path) {
+ return path == null || column.equals(path) ? field
+ : field.path("fields").path(path.substring(column.length() +
1));
+ }
+
+ private static boolean changesSourceValue(JsonNode field) {
+ return field.isMissingNode() || field.has("ignore_above") ||
field.hasNonNull("normalizer")
+ || field.hasNonNull("null_value") ||
field.path("ignore_malformed").asBoolean(false);
+ }
+
+ private static boolean roundsNumericSourceValue(JsonNode field) {
+ String type = field.path("type").asText();
+ return "scaled_float".equals(type) || "half_float".equals(type);
Review Comment:
[P2] Account for date-only source values in final-mode safety
Default `date` fields can also change the value Doris reads after this
source fallback. With the FE JVM timezone and SQL `time_zone` both
`Asia/Shanghai`, map `a` as a normalized keyword and `b` as `{"type":"date"}`,
then insert `{"a":"ok","b":"2024-01-01"}`. For `SELECT a,b FROM idx WHERE
b=CAST('2024-01-01 08:00:00' AS DATETIMEV2(0))`, `compatDefaultDate` sends
`2024-01-01T08:00:00+08:00`, matching ES's UTC-midnight date. Previously the
docvalue ISO timestamp was converted to session time and read as 08:00.
Removing `a`'s docvalue path now makes both readers parse the date-only source
through `CastToDatetimeV2`, yielding 00:00, yet `b` is neither a source string
nor one of these two numeric types, so the equality remains pruned. The
returned row fails WHERE; the converse midnight predicate can miss the source
row. Include this source-date representation difference in safety or normalize
the source parsing, with a non-UTC date-only regression. ES supports these
[default date valu
es](https://www.elastic.co/guide/en/elasticsearch/reference/8.19/date.html).
##########
fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsMappingUtils.java:
##########
@@ -150,12 +169,139 @@ public static EsFieldContext
resolveFieldContext(List<String> columnNames,
String fieldType = fieldObject.get("type").asText();
fieldContext.getColumn2typeMap().put(colName, fieldType);
resolveDateFields(fieldContext, fieldObject, colName, fieldType);
- resolveKeywordFields(fieldContext, fieldObject, colName,
fieldType);
resolveDocValuesFields(fieldContext, fieldObject, colName,
fieldType);
}
+ resolveSourceValueSafety(fieldContext, indexMapping, mappingType,
properties);
return fieldContext;
}
+ private static void resolveSourceValueSafety(EsFieldContext context,
String indexMapping, String mappingType,
+ ObjectNode schemaProperties) {
+ try {
+ // Mapping-induced source/index differences on any alias backing
index must prevent
+ // filtering or replacing source values through that path.
+ List<ObjectNode> roots = new ArrayList<>();
+ Set<String> columns = new
HashSet<>(context.getColumn2typeMap().keySet());
+ for (JsonNode index : MAPPER.readTree(indexMapping)) {
+ ObjectNode root = getRootSchema((ObjectNode)
index.get("mappings"), mappingType,
+ new ArrayList<>(), true);
+ roots.add(root);
+ }
+ // Use the same populated schema as inference, including
unprojected filter columns.
+ schemaProperties.fieldNames().forEachRemaining(columns::add);
+ // Filters can reference columns absent from the projection.
Resolve their query paths
+ // from the exposed schema too; the projection still controls
docvalue read eligibility.
+ for (String column : columns) {
+ ObjectNode field = (ObjectNode) schemaProperties.get(column);
+ resolveKeywordFields(context, field, column,
field.path("type").asText());
+ }
+ Set<String> unsafeDocValues = new HashSet<>();
+ for (ObjectNode root : roots) {
+ JsonNode properties = root.path("properties");
+ Set<String> copyTargets = new HashSet<>();
+ collectCopyTargets(properties, copyTargets);
+ for (String column : columns) {
+ JsonNode field = properties.path(column);
+ JsonNode queryField = fieldAtPath(field, column,
context.getFetchFieldsContext().get(column));
+ JsonNode readField = fieldAtPath(field, column,
context.getDocValueFieldsContext().get(column));
+ boolean sourceChanged =
filtersSourceColumn(root.path("_source"), column)
+ || root.path("runtime").has(column) ||
copyTargets.contains(column);
+ // ES mappings do not distinguish arrays from scalars.
Even a short equality can
+ // match one indexed array element while Doris reads the
entire source JSON string.
+ if (sourceChanged || changesSourceValue(field) ||
changesSourceValue(queryField)
+ || "text".equals(queryField.path("type").asText())
|| !isSearchable(queryField)) {
+ context.getSourceValueUnsafeFields().add(column);
+ }
+ // These mappings round indexed numbers while _source
retains the original value.
+ // Keep docvalues eligible: only a final source read makes
their predicates unsafe.
+ if (roundsNumericSourceValue(field) ||
roundsNumericSourceValue(queryField)) {
+ context.getSourceModeUnsafeFields().add(column);
+ }
+ // Query and docvalue paths may select different
multi-fields. An unused sibling's
+ // normalizer must not invalidate either path; only the
chosen read value matters.
+ if (sourceChanged || changesSourceValue(field) ||
changesSourceValue(readField)
+ || !readField.path("doc_values").asBoolean(true)
+ ||
DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(readField.path("type").asText())) {
+ // One backing without this docvalue path would return
missing values for its hits.
+ unsafeDocValues.add(column);
+ }
+ }
+ }
+
unsafeDocValues.forEach(context.getDocValueFieldsContext()::remove);
+ } catch (IOException e) {
+ throw new DorisConnectorException("Failed to parse ES mapping: " +
e.getMessage());
+ }
+ }
+
+ private static JsonNode fieldAtPath(JsonNode field, String column, String
path) {
+ return path == null || column.equals(path) ? field
+ : field.path("fields").path(path.substring(column.length() +
1));
+ }
+
+ private static boolean changesSourceValue(JsonNode field) {
+ return field.isMissingNode() || field.has("ignore_above") ||
field.hasNonNull("normalizer")
+ || field.hasNonNull("null_value") ||
field.path("ignore_malformed").asBoolean(false);
+ }
+
+ private static boolean roundsNumericSourceValue(JsonNode field) {
+ String type = field.path("type").asText();
+ return "scaled_float".equals(type) || "half_float".equals(type);
+ }
+
+ private static boolean isSearchable(JsonNode field) {
+ // Disabling the inverted index does not change the value. ES can
still search keyword
+ // docvalues; only a field with neither representation needs a local
predicate.
+ return field.path("index").asBoolean(true)
+ || ("keyword".equals(field.path("type").asText()) &&
field.path("doc_values").asBoolean(true));
+ }
+
+ private static boolean filtersSourceColumn(JsonNode source, String column)
{
+ JsonNode includes = source.path("includes");
+ return !source.path("enabled").asBoolean(true)
Review Comment:
[P2] Recognize disabled source mode in the mapping safety check
ES 8.11 accepts `_source:{"mode":"disabled"}` and serializes it without an
`enabled` key; its [source
mapper](https://github.com/elastic/elasticsearch/blob/v8.11.3/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java)
makes this equivalent to disabled source. This check instead defaults the
missing `enabled` to true. Map `a` as a normalized keyword and `b` as `long`
under that source mode, and index `{"a":"ok","b":1}`. With docvalue scans
enabled, `SELECT a,b FROM idx WHERE b=1` previously returned `(ok,1)` from
docvalues. The new normalizer check removes only `a`'s docvalue path, forcing a
source request with no docvalue fields, while ordinary numeric `b` stays safe
and its equality is pruned. ES returns neither source nor fields, so both
readers insert `(NULL,NULL)`, violating the predicate. Apply the
disabled-source restrictions for `mode=disabled` as well as `enabled=false`,
including all alias backings, and add this mapping form to the regression
coverag
e.
##########
fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsMappingUtils.java:
##########
@@ -150,12 +169,139 @@ public static EsFieldContext
resolveFieldContext(List<String> columnNames,
String fieldType = fieldObject.get("type").asText();
fieldContext.getColumn2typeMap().put(colName, fieldType);
resolveDateFields(fieldContext, fieldObject, colName, fieldType);
- resolveKeywordFields(fieldContext, fieldObject, colName,
fieldType);
resolveDocValuesFields(fieldContext, fieldObject, colName,
fieldType);
}
+ resolveSourceValueSafety(fieldContext, indexMapping, mappingType,
properties);
return fieldContext;
}
+ private static void resolveSourceValueSafety(EsFieldContext context,
String indexMapping, String mappingType,
+ ObjectNode schemaProperties) {
+ try {
+ // Mapping-induced source/index differences on any alias backing
index must prevent
+ // filtering or replacing source values through that path.
+ List<ObjectNode> roots = new ArrayList<>();
+ Set<String> columns = new
HashSet<>(context.getColumn2typeMap().keySet());
+ for (JsonNode index : MAPPER.readTree(indexMapping)) {
+ ObjectNode root = getRootSchema((ObjectNode)
index.get("mappings"), mappingType,
+ new ArrayList<>(), true);
+ roots.add(root);
+ }
+ // Use the same populated schema as inference, including
unprojected filter columns.
+ schemaProperties.fieldNames().forEachRemaining(columns::add);
+ // Filters can reference columns absent from the projection.
Resolve their query paths
+ // from the exposed schema too; the projection still controls
docvalue read eligibility.
+ for (String column : columns) {
+ ObjectNode field = (ObjectNode) schemaProperties.get(column);
+ resolveKeywordFields(context, field, column,
field.path("type").asText());
+ }
+ Set<String> unsafeDocValues = new HashSet<>();
+ for (ObjectNode root : roots) {
+ JsonNode properties = root.path("properties");
+ Set<String> copyTargets = new HashSet<>();
+ collectCopyTargets(properties, copyTargets);
+ for (String column : columns) {
+ JsonNode field = properties.path(column);
+ JsonNode queryField = fieldAtPath(field, column,
context.getFetchFieldsContext().get(column));
+ JsonNode readField = fieldAtPath(field, column,
context.getDocValueFieldsContext().get(column));
+ boolean sourceChanged =
filtersSourceColumn(root.path("_source"), column)
+ || root.path("runtime").has(column) ||
copyTargets.contains(column);
+ // ES mappings do not distinguish arrays from scalars.
Even a short equality can
+ // match one indexed array element while Doris reads the
entire source JSON string.
+ if (sourceChanged || changesSourceValue(field) ||
changesSourceValue(queryField)
+ || "text".equals(queryField.path("type").asText())
|| !isSearchable(queryField)) {
+ context.getSourceValueUnsafeFields().add(column);
+ }
+ // These mappings round indexed numbers while _source
retains the original value.
+ // Keep docvalues eligible: only a final source read makes
their predicates unsafe.
+ if (roundsNumericSourceValue(field) ||
roundsNumericSourceValue(queryField)) {
+ context.getSourceModeUnsafeFields().add(column);
+ }
+ // Query and docvalue paths may select different
multi-fields. An unused sibling's
+ // normalizer must not invalidate either path; only the
chosen read value matters.
+ if (sourceChanged || changesSourceValue(field) ||
changesSourceValue(readField)
+ || !readField.path("doc_values").asBoolean(true)
+ ||
DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(readField.path("type").asText())) {
+ // One backing without this docvalue path would return
missing values for its hits.
+ unsafeDocValues.add(column);
+ }
+ }
+ }
+
unsafeDocValues.forEach(context.getDocValueFieldsContext()::remove);
Review Comment:
[P2] Preserve supported boolean reads when forcing source fallback
Removing one field's docvalue path also changes how every sibling is
decoded. Map `a` as a keyword with a lowercase normalizer and `b` as `boolean`,
then insert `{"a":"ok","b":""}`. Elasticsearch explicitly [accepts the empty
string as
false](https://www.elastic.co/guide/en/elasticsearch/reference/8.19/boolean.html).
With docvalue scans enabled, `SELECT a,b FROM idx` previously read the Boolean
docvalue and returned `(ok,false)`. This new normalizer check removes `a`'s
path and forces source for both columns. `ScrollParser` then passes the empty
source string to `StringParser::string_to_bool`, which reports PARSE_FAILURE
for length zero and aborts the query on both readers; the `SELECT b`
docvalue-only control still succeeds. No WHERE condition is needed, so adding
residual guards cannot repair this regression. Ensure the newly forced source
path can materialize supported ES boolean values, or preserve a compatible read
path, and cover this sibling/control case before switching th
e scan mode.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
