wangxianghu commented on code in PR #19081: URL: https://github.com/apache/hudi/pull/19081#discussion_r3509951598
########## hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/ttl/strategy/KeepByEventTimeStrategy.java: ########## @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.action.ttl.strategy; + +import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator; +import org.apache.hudi.common.util.PartitionPathEncodeUtils; +import org.apache.hudi.config.HoodieTTLConfig; +import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.hudi.table.HoodieTable; + +import lombok.extern.slf4j.Slf4j; + +import java.text.ParseException; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Event-time based partition TTL strategy: lifetime is read from the partition path, not from + * commit metadata. Late-arriving writes into an old partition do not extend its lifetime, and + * backfilled historic partitions are still considered old. Compare with {@link KeepByTimeStrategy} + * (last commit time) and {@link KeepByCreationTimeStrategy} (creation commit time). + * + * <h3>Supported path shapes</h3> + * The first-class, tested shapes are day and hour granularity. Each example shows the partition + * path on the left and the required {@code timeSegStartIndex} on the right (see + * <i>Locating the time block</i> below); non-time segments may appear before and/or after the + * time block. + * <ul> + * <li>Day, {@code format=yyyy-MM-dd} + * <ul> + * <li>time only: {@code 2026-06-27}, {@code dt=2026-06-27} — timeSegStartIndex {@code 0}</li> + * <li>prefix + time: {@code region=us/2026-06-27}, {@code region=us/dt=2026-06-27} — timeSegStartIndex {@code 1}</li> + * <li>time + suffix: {@code 2026-06-27/source=app}, {@code dt=2026-06-27/source=app} — timeSegStartIndex {@code 0}</li> + * <li>prefix + time + suffix: {@code region=us/dt=2026-06-27/source=app} — timeSegStartIndex {@code 1}</li> + * </ul> + * </li> + * <li>Day, {@code format=yyyyMMdd} + * <ul> + * <li>time only: {@code 20260627}, {@code dt=20260627} — timeSegStartIndex {@code 0}</li> + * <li>prefix + time: {@code region=us/20260627}, {@code region=us/dt=20260627} — timeSegStartIndex {@code 1}</li> + * <li>time + suffix: {@code 20260627/source=app}, {@code dt=20260627/source=app} — timeSegStartIndex {@code 0}</li> + * <li>prefix + time + suffix: {@code region=us/dt=20260627/source=app} — timeSegStartIndex {@code 1}</li> + * </ul> + * </li> + * <li>Hour, {@code format=yyyy-MM-dd/HH} + * <ul> + * <li>time only: {@code 2026-06-27/12}, {@code dt=2026-06-27/hh=12} — timeSegStartIndex {@code 0}</li> + * <li>prefix + time: {@code region=us/2026-06-27/12}, {@code region=us/dt=2026-06-27/hh=12} — timeSegStartIndex {@code 1}</li> + * <li>time + suffix: {@code 2026-06-27/12/source=app}, {@code dt=2026-06-27/hh=12/source=app} — timeSegStartIndex {@code 0}</li> + * <li>prefix + time + suffix: {@code region=us/dt=2026-06-27/hh=12/source=app} — timeSegStartIndex {@code 1}</li> + * </ul> + * </li> + * <li>Hour, {@code format=yyyyMMdd/HH} + * <ul> + * <li>time only: {@code 20260627/12}, {@code dt=20260627/hh=12} — timeSegStartIndex {@code 0}</li> + * <li>prefix + time: {@code region=us/20260627/12}, {@code region=us/dt=20260627/hh=12} — timeSegStartIndex {@code 1}</li> + * <li>time + suffix: {@code 20260627/12/source=app}, {@code dt=20260627/hh=12/source=app} — timeSegStartIndex {@code 0}</li> + * <li>prefix + time + suffix: {@code region=us/dt=20260627/hh=12/source=app} — timeSegStartIndex {@code 1}</li> + * </ul> + * </li> + * </ul> + * Hive-style key names are not constrained: {@code dt=}, {@code day=}, {@code event_date=}, + * {@code hh=}, {@code hour=} all work; only the value after {@code =} is parsed. + * <p> + * Any {@link java.time.format.DateTimeFormatter} pattern works as long as the resulting + * {@link java.time.temporal.TemporalAccessor} can be resolved to either an {@link java.time.Instant} + * or a {@link java.time.LocalDate} (day-only patterns are anchored at UTC start-of-day). A + * {@code /} in the pattern means the time value spans that many consecutive path segments. + * Patterns missing day-of-month, e.g. month-only {@code yyyy-MM}, cannot be resolved and will + * raise the standard parse-failure error at runtime. + * + * <h3>Locating the time block</h3> + * The time block must occupy a <i>contiguous</i> segment range of the partition path. Only the + * segments before it count toward {@code timeSegStartIndex}; segments after are ignored. Interleaved + * layouts such as {@code dt=20260627/source=app/hh=12} are not supported -- the time block must + * be in one piece. + * + * <h3>Time zone</h3> + * Both the partition's event time and the cutoff derived from {@code instantTime} are interpreted + * in UTC. Set {@code hoodie.table.timeline.timezone=UTC} so the timeline writes instants under the + * same convention; otherwise the cutoff drifts by the JVM's UTC offset -- a boundary effect at + * day granularity, a full-offset shift at hour granularity. + * + * <h3>Configuration</h3> + * All three knobs come with defaults, so a table whose partition path is purely a date in + * {@code yyyy-MM-dd} form works out of the box. + * <ul> + * <li>{@link org.apache.hudi.config.HoodieTTLConfig#EVENT_TIME_FORMAT} — date-time pattern of + * the time block in the partition path. Default {@code yyyy-MM-dd}. A {@code /} in the + * pattern means the time block spans that many consecutive segments.</li> + * <li>{@link org.apache.hudi.config.HoodieTTLConfig#EVENT_TIME_SEGMENT_START_INDEX} — + * 0-based index of the first segment that carries the time block. Default {@code 0}. + * Raise it when non-time segments come before the time block (see examples above).</li> + * <li>{@link org.apache.hudi.config.HoodieTTLConfig#EVENT_TIME_DELETE_HIVE_DEFAULT_PARTITION} — + * whether to treat partitions whose time block contains {@code __HIVE_DEFAULT_PARTITION__} + * in <i>any</i> segment as expired. Such a partition has an undefined event time (one of + * its event-time columns was {@code NULL}, so the row cannot be placed on the configured + * time axis) and falls outside the normal cutoff comparison. Default {@code false}, i.e. + * such partitions are skipped with a WARN and the user keeps explicit control over them. + * Applies to both single-segment formats (e.g. {@code dt=__HIVE_DEFAULT_PARTITION__}) and + * multi-segment ones (e.g. {@code dt=2026-06-28/hh=__HIVE_DEFAULT_PARTITION__}).</li> + * </ul> + */ +@Slf4j Review Comment: lots of files in hudi-client-common use Slf4j, maybe Slf4j is the right way -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
