prashantwason commented on code in PR #17550:
URL: https://github.com/apache/hudi/pull/17550#discussion_r2730315751
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCleanConfig.java:
##########
@@ -201,6 +201,11 @@ public class HoodieCleanConfig extends HoodieConfig {
+ "table receives updates/deletes. Another reason to turn this on,
would be to ensure data residing in bootstrap "
+ "base files are also physically deleted, to comply with data
privacy enforcement processes.");
+ public static final ConfigProperty<String> CLEANER_PARTITION_FILTER_REGEX =
ConfigProperty
+ .key("hoodie.cleaner.partition.filter.regex")
Review Comment:
Done
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java:
##########
@@ -260,11 +260,23 @@ private Stream<String>
getPartitionsForInstants(HoodieInstant instant) {
*/
private List<String> getPartitionPathsForFullCleaning() {
// Go to brute force mode of scanning all partitions
+ List<String> allPartitionPaths;
try {
- return hoodieTable.getMetadataTable().getAllPartitionPaths();
+ allPartitionPaths =
hoodieTable.getMetadataTable().getAllPartitionPaths();
} catch (IOException ioe) {
throw new HoodieIOException("Fetching all partitions failed ", ioe);
}
+
+ if (!config.getCleanerPartitionRegex().isEmpty()) {
+ if (config.incrementalCleanerModeEnabled()) {
+ throw new IllegalArgumentException("Incremental Cleaning mode is
enabled. Partition regex for clean cannot be used.");
+ }
+ LOG.info("Restricting partition to clean using regex " +
config.getCleanerPartitionRegex());
+ allPartitionPaths = allPartitionPaths.stream().filter(p ->
p.matches(config.getCleanerPartitionRegex()))
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
