rdblue commented on code in PR #5215:
URL: https://github.com/apache/iceberg/pull/5215#discussion_r916019988
##########
core/src/main/java/org/apache/iceberg/MetricsConfig.java:
##########
@@ -136,50 +131,58 @@ public static MetricsConfig forPositionDelete(Table
table) {
}
/**
- * Generate a MetricsConfig for all columns based on overrides, sortOrder,
and defaultMode.
+ * Generate a MetricsConfig for all columns based on overrides, schema, and
sort order.
+ *
* @param props will be read for metrics overrides
(write.metadata.metrics.column.*) and default
* (write.metadata.metrics.default)
+ * @param schema table schema
* @param order sort order columns, will be promoted to truncate(16)
- * @param defaultMode default, if not set by user property
* @return metrics configuration
*/
- private static MetricsConfig from(Map<String, String> props, SortOrder
order, String defaultMode) {
+ private static MetricsConfig from(Map<String, String> props, Schema schema,
SortOrder order) {
+ int maxInferredDefaultColumns = PropertyUtil.propertyAsInt(props,
+ TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS,
+ TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT);
Map<String, MetricsMode> columnModes = Maps.newHashMap();
// Handle user override of default mode
- MetricsMode finalDefaultMode;
- String defaultModeAsString =
props.getOrDefault(DEFAULT_WRITE_METRICS_MODE, defaultMode);
- try {
- finalDefaultMode = MetricsModes.fromString(defaultModeAsString);
- } catch (IllegalArgumentException err) {
- // User override was invalid, log the error and use the default
- LOG.warn("Ignoring invalid default metrics mode: {}",
defaultModeAsString, err);
- finalDefaultMode = MetricsModes.fromString(defaultMode);
+ MetricsMode defaultMode;
+ String configuredDefault = props.get(DEFAULT_WRITE_METRICS_MODE);
+ if (configuredDefault != null) {
+ // a user-configured default mode is applied for all columns
+ defaultMode = parseMode(configuredDefault, DEFAULT_MODE, "default");
+
+ } else if (schema == null || schema.columns().size() <=
maxInferredDefaultColumns) {
+ // there are less than the inferred limit, so the default is used
everywhere
+ defaultMode = DEFAULT_MODE;
+
+ } else {
+ // a inferred default mode is applied to the first few columns, up to
the limit
+ Schema subSchema = new Schema(schema.columns().subList(0,
maxInferredDefaultColumns));
+ for (Integer id : TypeUtil.getProjectedIds(subSchema)) {
+ columnModes.put(subSchema.findColumnName(id), DEFAULT_MODE);
+ }
+
+ // all other columns don't use metrics
+ defaultMode = MetricsModes.None.get();
}
// First set sorted column with sorted column default (can be overridden
by user)
- MetricsMode sortedColDefaultMode =
sortedColumnDefaultMode(finalDefaultMode);
+ MetricsMode sortedColDefaultMode = sortedColumnDefaultMode(defaultMode);
Set<String> sortedCols = SortOrderUtil.orderPreservingSortedColumns(order);
sortedCols.forEach(sc -> columnModes.put(sc, sortedColDefaultMode));
// Handle user overrides of defaults
- MetricsMode finalDefaultModeVal = finalDefaultMode;
- props.keySet().stream()
- .filter(key -> key.startsWith(METRICS_MODE_COLUMN_CONF_PREFIX))
- .forEach(key -> {
- String columnAlias =
key.replaceFirst(METRICS_MODE_COLUMN_CONF_PREFIX, "");
- MetricsMode mode;
- try {
- mode = MetricsModes.fromString(props.get(key));
- } catch (IllegalArgumentException err) {
- // Mode was invalid, log the error and use the default
- LOG.warn("Ignoring invalid metrics mode for column {}: {}",
columnAlias, props.get(key), err);
- mode = finalDefaultModeVal;
- }
- columnModes.put(columnAlias, mode);
- });
+ MetricsMode finalDefaultModeVal = defaultMode;
Review Comment:
Fixed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
