jt2594838 commented on code in PR #16789: URL: https://github.com/apache/iotdb/pull/16789#discussion_r2579408723
########## iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/TreePatternPruningTest.java: ########## @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.pattern; + +import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.WithExclusionIoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.WithExclusionTreePattern; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeException; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; + +public class TreePatternPruningTest { + + @Test + public void testUnionInternalPruning_Cover() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.db.d1.*,root.db.d1.s1"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue("Should be IoTDBTreePattern", result instanceof IoTDBTreePattern); + Assert.assertEquals("root.db.d1.*", result.getPattern()); + } + + @Test + public void testUnionInternalPruning_Duplicates() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.db.d1,root.db.d1"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue(result instanceof IoTDBTreePattern); + Assert.assertEquals("root.db.d1", result.getPattern()); + } + + @Test + public void testInclusionPrunedByExclusion_Partial() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.sg.d1,root.sg.d2"); + put(PipeSourceConstant.SOURCE_PATH_EXCLUSION_KEY, "root.sg.d1"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue(result instanceof WithExclusionIoTDBTreePattern); + final WithExclusionIoTDBTreePattern exclusionPattern = (WithExclusionIoTDBTreePattern) result; + + Assert.assertEquals("root.sg.d2", exclusionPattern.getInclusionPattern().getPattern()); + Assert.assertEquals("root.sg.d1", exclusionPattern.getExclusionPattern().getPattern()); + } Review Comment: How about just leaving a single "root.sg.d2" as IoTDBTreePattern? ########## iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/TreePatternPruningTest.java: ########## @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.pattern; + +import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.WithExclusionIoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.WithExclusionTreePattern; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeException; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; + +public class TreePatternPruningTest { + + @Test + public void testUnionInternalPruning_Cover() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.db.d1.*,root.db.d1.s1"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue("Should be IoTDBTreePattern", result instanceof IoTDBTreePattern); + Assert.assertEquals("root.db.d1.*", result.getPattern()); + } + + @Test + public void testUnionInternalPruning_Duplicates() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.db.d1,root.db.d1"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue(result instanceof IoTDBTreePattern); + Assert.assertEquals("root.db.d1", result.getPattern()); + } + + @Test + public void testInclusionPrunedByExclusion_Partial() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.sg.d1,root.sg.d2"); + put(PipeSourceConstant.SOURCE_PATH_EXCLUSION_KEY, "root.sg.d1"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue(result instanceof WithExclusionIoTDBTreePattern); + final WithExclusionIoTDBTreePattern exclusionPattern = (WithExclusionIoTDBTreePattern) result; + + Assert.assertEquals("root.sg.d2", exclusionPattern.getInclusionPattern().getPattern()); + Assert.assertEquals("root.sg.d1", exclusionPattern.getExclusionPattern().getPattern()); + } + + @Test + public void testInclusionPrunedByExclusion_FullCoverage_Exception() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.sg.d1"); + put(PipeSourceConstant.SOURCE_PATH_EXCLUSION_KEY, "root.sg.**"); + } + }); + + try { + TreePattern.parsePipePatternFromSourceParameters(params); + Assert.fail("Should throw PipeException because Exclusion fully covers Inclusion"); + } catch (final PipeException ignored) { + // Expected exception + } + } + + @Test + public void testComplexPruning() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.sg.A,root.sg.B,root.sg.A.sub"); + put(PipeSourceConstant.SOURCE_PATH_EXCLUSION_KEY, "root.sg.A,root.sg.A.**"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue(result instanceof WithExclusionIoTDBTreePattern); + final WithExclusionIoTDBTreePattern excPattern = (WithExclusionIoTDBTreePattern) result; + + Assert.assertEquals("root.sg.B", excPattern.getInclusionPattern().getPattern()); + } + + @Test + public void testComplexPruning_Prefix() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATTERN_KEY, "root.sg.A,root.sg.B,root.sg.A.sub"); + put(PipeSourceConstant.SOURCE_PATTERN_EXCLUSION_KEY, "root.sg.A"); + put(PipeSourceConstant.SOURCE_PATTERN_FORMAT_KEY, "prefix"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue(result instanceof WithExclusionTreePattern); + final WithExclusionTreePattern excPattern = (WithExclusionTreePattern) result; + + Assert.assertEquals("root.sg.B", excPattern.getInclusionPattern().getPattern()); + } + + @Test + public void testUnionPreservedWhenNotCovered() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.sg.d1,root.sg.d2"); + put(PipeSourceConstant.SOURCE_PATH_EXCLUSION_KEY, "root.other"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue(result instanceof WithExclusionIoTDBTreePattern); + final WithExclusionIoTDBTreePattern excResult = (WithExclusionIoTDBTreePattern) result; + + Assert.assertTrue(excResult.getInclusionPattern() instanceof UnionIoTDBTreePattern); + final UnionIoTDBTreePattern unionInc = (UnionIoTDBTreePattern) excResult.getInclusionPattern(); + Assert.assertEquals(2, unionInc.getPatterns().size()); + } Review Comment: If not covered, why still keep the exclusion? ########## iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java: ########## @@ -274,65 +286,145 @@ public static TreePattern parsePatternFromString( } /** - * A private helper method to parse a set of 'path' and 'pattern' keys into a single union - * TreePattern. This contains the original logic of parsePipePatternFromSourceParameters. - * - * @param sourceParameters The source parameters. - * @param isTreeModelDataAllowedToBeCaptured Flag for TreePattern constructor. - * @param extractorPathKey Key for extractor path (e.g., "extractor.path"). - * @param sourcePathKey Key for source path (e.g., "source.path"). - * @param extractorPatternKey Key for extractor pattern (e.g., "extractor.pattern"). - * @param sourcePatternKey Key for source pattern (e.g., "source.pattern"). - * @param defaultPattern The pattern to return if both path and pattern are null. If this - * parameter is null, this method returns null. - * @return The parsed TreePattern, or defaultPattern, or null if defaultPattern is null and no - * patterns are specified. + * Helper method to parse pattern parameters into a list of patterns without creating the Union + * object immediately. */ - private static TreePattern parsePatternUnion( + private static List<TreePattern> parsePatternList( final PipeParameters sourceParameters, final boolean isTreeModelDataAllowedToBeCaptured, final String extractorPathKey, final String sourcePathKey, final String extractorPatternKey, - final String sourcePatternKey, - final TreePattern defaultPattern) { + final String sourcePatternKey) { final String path = sourceParameters.getStringByKeys(extractorPathKey, sourcePathKey); final String pattern = sourceParameters.getStringByKeys(extractorPatternKey, sourcePatternKey); - // 1. If both "source.path" and "source.pattern" are specified, their union will be used. - if (path != null && pattern != null) { - final List<TreePattern> result = new ArrayList<>(); - // Parse "source.path" as IoTDB-style path. + final List<TreePattern> result = new ArrayList<>(); + + if (path != null) { result.addAll( parseMultiplePatterns( path, p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p))); - // Parse "source.pattern" using the helper method. + } + + if (pattern != null) { result.addAll( parsePatternsFromPatternParameter( pattern, sourceParameters, isTreeModelDataAllowedToBeCaptured)); - return buildUnionPattern(isTreeModelDataAllowedToBeCaptured, result); } - // 2. If only "source.path" is specified, it will be interpreted as an IoTDB-style path. - if (path != null) { - return buildUnionPattern( - isTreeModelDataAllowedToBeCaptured, - parseMultiplePatterns( - path, p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p))); + return result; + } + + /** + * Removes patterns from the list that are covered by other patterns in the same list. For + * example, if "root.**" and "root.db.**" are present, "root.db.**" is removed. + */ + private static List<TreePattern> optimizePatterns(final List<TreePattern> patterns) { + if (patterns == null || patterns.isEmpty()) { + return new ArrayList<>(); + } + if (patterns.size() == 1) { + return patterns; } - // 3. If only "source.pattern" is specified, parse it using the helper method. - if (pattern != null) { - return buildUnionPattern( - isTreeModelDataAllowedToBeCaptured, - parsePatternsFromPatternParameter( - pattern, sourceParameters, isTreeModelDataAllowedToBeCaptured)); + final List<TreePattern> optimized = new ArrayList<>(); + // Determine coverage using base paths + for (int i = 0; i < patterns.size(); i++) { + final TreePattern current = patterns.get(i); + boolean isCoveredByOther = false; + + for (int j = 0; j < patterns.size(); j++) { + if (i == j) { + continue; + } + final TreePattern other = patterns.get(j); + + // If 'other' covers 'current', 'current' is redundant. + // Note: if they mutually cover each other (duplicates), we must ensure we keep one. + // We use index comparison to break ties for exact duplicates. + if (covers(other, current)) { + if (covers(current, other)) { + // Both cover each other (likely identical). Keep the one with smaller index. + if (j < i) { + isCoveredByOther = true; + break; + } + } else { + // Strict coverage + isCoveredByOther = true; + break; + } + } + } + + if (!isCoveredByOther) { + optimized.add(current); + } + } + return optimized; + } + + /** + * Prunes patterns from the inclusion list that are fully covered by ANY pattern in the exclusion + * list. + */ + private static List<TreePattern> pruneInclusionPatterns( + final List<TreePattern> inclusion, final List<TreePattern> exclusion) { + if (inclusion == null || inclusion.isEmpty()) { + return new ArrayList<>(); + } + if (exclusion == null || exclusion.isEmpty()) { + return inclusion; } - // 4. If neither "source.path" nor "source.pattern" is specified, - // return the provided default pattern (which may be null). - return defaultPattern; + final List<TreePattern> prunedInclusion = new ArrayList<>(); + for (final TreePattern inc : inclusion) { + boolean isFullyExcluded = false; + for (final TreePattern exc : exclusion) { + if (covers(exc, inc)) { + isFullyExcluded = true; + break; + } + } + if (!isFullyExcluded) { + prunedInclusion.add(inc); + } + } + return prunedInclusion; + } + + /** Checks if 'coverer' pattern fully covers 'coveree' pattern. */ + private static boolean covers(final TreePattern coverer, final TreePattern coveree) { + try { + final List<PartialPath> covererPaths = coverer.getBaseInclusionPaths(); + final List<PartialPath> covereePaths = coveree.getBaseInclusionPaths(); + + if (covererPaths.isEmpty() || covereePaths.isEmpty()) { + return false; + } + + // Logic: For 'coverer' to cover 'coveree', ALL paths in 'coveree' must be included + // by at least one path in 'coverer'. + for (final PartialPath sub : covereePaths) { + boolean isSubCovered = false; + for (final PartialPath sup : covererPaths) { + if (sup.include(sub)) { + isSubCovered = true; + break; + } + } + if (!isSubCovered) { + return false; + } + } + return true; + } catch (final Exception e) { + // In case of path parsing errors or unsupported operations, assume no coverage + // to be safe and avoid aggressive pruning. + return false; Review Comment: Print a warn log with the pattterns ########## iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/TreePatternPruningTest.java: ########## @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.pattern; + +import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.WithExclusionIoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.WithExclusionTreePattern; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeException; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; + +public class TreePatternPruningTest { + + @Test + public void testUnionInternalPruning_Cover() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.db.d1.*,root.db.d1.s1"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue("Should be IoTDBTreePattern", result instanceof IoTDBTreePattern); + Assert.assertEquals("root.db.d1.*", result.getPattern()); + } + + @Test + public void testUnionInternalPruning_Duplicates() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.db.d1,root.db.d1"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue(result instanceof IoTDBTreePattern); + Assert.assertEquals("root.db.d1", result.getPattern()); + } + + @Test + public void testInclusionPrunedByExclusion_Partial() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.sg.d1,root.sg.d2"); + put(PipeSourceConstant.SOURCE_PATH_EXCLUSION_KEY, "root.sg.d1"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue(result instanceof WithExclusionIoTDBTreePattern); + final WithExclusionIoTDBTreePattern exclusionPattern = (WithExclusionIoTDBTreePattern) result; + + Assert.assertEquals("root.sg.d2", exclusionPattern.getInclusionPattern().getPattern()); + Assert.assertEquals("root.sg.d1", exclusionPattern.getExclusionPattern().getPattern()); + } + + @Test + public void testInclusionPrunedByExclusion_FullCoverage_Exception() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.sg.d1"); + put(PipeSourceConstant.SOURCE_PATH_EXCLUSION_KEY, "root.sg.**"); + } + }); + + try { + TreePattern.parsePipePatternFromSourceParameters(params); + Assert.fail("Should throw PipeException because Exclusion fully covers Inclusion"); + } catch (final PipeException ignored) { + // Expected exception + } + } + + @Test + public void testComplexPruning() { + final PipeParameters params = + new PipeParameters( + new HashMap<String, String>() { + { + put(PipeSourceConstant.SOURCE_PATH_KEY, "root.sg.A,root.sg.B,root.sg.A.sub"); + put(PipeSourceConstant.SOURCE_PATH_EXCLUSION_KEY, "root.sg.A,root.sg.A.**"); + } + }); + + final TreePattern result = TreePattern.parsePipePatternFromSourceParameters(params); + + Assert.assertTrue(result instanceof WithExclusionIoTDBTreePattern); + final WithExclusionIoTDBTreePattern excPattern = (WithExclusionIoTDBTreePattern) result; + + Assert.assertEquals("root.sg.B", excPattern.getInclusionPattern().getPattern()); + } Review Comment: Similarly, the result should only contain an inclusion "root.sg.B" and no exclusion. ########## iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java: ########## @@ -140,60 +140,72 @@ public static TreePattern parsePipePatternFromSourceParameters( final boolean isTreeModelDataAllowedToBeCaptured = isTreeModelDataAllowToBeCaptured(sourceParameters); - // 1. Define the default inclusion pattern (matches all, "root.**") - // This is used if no inclusion patterns are specified. - final TreePattern defaultInclusionPattern = - buildUnionPattern( - isTreeModelDataAllowedToBeCaptured, - Collections.singletonList( - new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, null))); - - // 2. Parse INCLUSION patterns using the helper - final TreePattern inclusionPattern = - parsePatternUnion( + // 1. Parse INCLUSION patterns into a list + List<TreePattern> inclusionPatterns = + parsePatternList( sourceParameters, isTreeModelDataAllowedToBeCaptured, - // 'path' keys (IoTDB wildcard) EXTRACTOR_PATH_KEY, SOURCE_PATH_KEY, - // 'pattern' keys (Prefix or IoTDB via format) EXTRACTOR_PATTERN_KEY, - SOURCE_PATTERN_KEY, - // Default pattern if no keys are found - defaultInclusionPattern); + SOURCE_PATTERN_KEY); + + // If no inclusion patterns are specified, use default "root.**" + if (inclusionPatterns.isEmpty()) { + inclusionPatterns = + new ArrayList<>( + Collections.singletonList( + new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, null))); + } - // 3. Parse EXCLUSION patterns using the helper - final TreePattern exclusionPattern = - parsePatternUnion( + // 2. Parse EXCLUSION patterns into a list + List<TreePattern> exclusionPatterns = + parsePatternList( sourceParameters, isTreeModelDataAllowedToBeCaptured, - // 'path.exclusion' keys (IoTDB wildcard) EXTRACTOR_PATH_EXCLUSION_KEY, SOURCE_PATH_EXCLUSION_KEY, - // 'pattern.exclusion' keys (Prefix) EXTRACTOR_PATTERN_EXCLUSION_KEY, - SOURCE_PATTERN_EXCLUSION_KEY, - // Default for exclusion is "match nothing" (null) - null); - - // 4. Combine inclusion and exclusion - if (exclusionPattern == null) { - // No exclusion defined, return the inclusion pattern directly - return inclusionPattern; - } else { - // If both inclusion and exclusion patterns support IoTDB operations, - // use the specialized ExclusionIoTDBTreePattern - if (inclusionPattern instanceof IoTDBTreePatternOperations - && exclusionPattern instanceof IoTDBTreePatternOperations) { - return new WithExclusionIoTDBTreePattern( - isTreeModelDataAllowedToBeCaptured, - (IoTDBTreePatternOperations) inclusionPattern, - (IoTDBTreePatternOperations) exclusionPattern); - } - // Both are defined, wrap them in an ExclusionTreePattern - return new WithExclusionTreePattern( - isTreeModelDataAllowedToBeCaptured, inclusionPattern, exclusionPattern); + SOURCE_PATTERN_EXCLUSION_KEY); + + // 3. Optimize the lists: remove redundant patterns (e.g., if "root.**" exists, "root.db" is + // redundant) + inclusionPatterns = optimizePatterns(inclusionPatterns); + exclusionPatterns = optimizePatterns(exclusionPatterns); + + // 4. Prune inclusion patterns: if an inclusion pattern is fully covered by an exclusion + // pattern, remove it + inclusionPatterns = pruneInclusionPatterns(inclusionPatterns, exclusionPatterns); Review Comment: Also remove exclusion patterns that do not cover any inclusions anymore? ########## iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java: ########## @@ -274,65 +286,145 @@ public static TreePattern parsePatternFromString( } /** - * A private helper method to parse a set of 'path' and 'pattern' keys into a single union - * TreePattern. This contains the original logic of parsePipePatternFromSourceParameters. - * - * @param sourceParameters The source parameters. - * @param isTreeModelDataAllowedToBeCaptured Flag for TreePattern constructor. - * @param extractorPathKey Key for extractor path (e.g., "extractor.path"). - * @param sourcePathKey Key for source path (e.g., "source.path"). - * @param extractorPatternKey Key for extractor pattern (e.g., "extractor.pattern"). - * @param sourcePatternKey Key for source pattern (e.g., "source.pattern"). - * @param defaultPattern The pattern to return if both path and pattern are null. If this - * parameter is null, this method returns null. - * @return The parsed TreePattern, or defaultPattern, or null if defaultPattern is null and no - * patterns are specified. + * Helper method to parse pattern parameters into a list of patterns without creating the Union + * object immediately. */ - private static TreePattern parsePatternUnion( + private static List<TreePattern> parsePatternList( final PipeParameters sourceParameters, final boolean isTreeModelDataAllowedToBeCaptured, final String extractorPathKey, final String sourcePathKey, final String extractorPatternKey, - final String sourcePatternKey, - final TreePattern defaultPattern) { + final String sourcePatternKey) { final String path = sourceParameters.getStringByKeys(extractorPathKey, sourcePathKey); final String pattern = sourceParameters.getStringByKeys(extractorPatternKey, sourcePatternKey); - // 1. If both "source.path" and "source.pattern" are specified, their union will be used. - if (path != null && pattern != null) { - final List<TreePattern> result = new ArrayList<>(); - // Parse "source.path" as IoTDB-style path. + final List<TreePattern> result = new ArrayList<>(); + + if (path != null) { result.addAll( parseMultiplePatterns( path, p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p))); - // Parse "source.pattern" using the helper method. + } + + if (pattern != null) { result.addAll( parsePatternsFromPatternParameter( pattern, sourceParameters, isTreeModelDataAllowedToBeCaptured)); - return buildUnionPattern(isTreeModelDataAllowedToBeCaptured, result); } - // 2. If only "source.path" is specified, it will be interpreted as an IoTDB-style path. - if (path != null) { - return buildUnionPattern( - isTreeModelDataAllowedToBeCaptured, - parseMultiplePatterns( - path, p -> new IoTDBTreePattern(isTreeModelDataAllowedToBeCaptured, p))); + return result; + } + + /** + * Removes patterns from the list that are covered by other patterns in the same list. For + * example, if "root.**" and "root.db.**" are present, "root.db.**" is removed. + */ + private static List<TreePattern> optimizePatterns(final List<TreePattern> patterns) { + if (patterns == null || patterns.isEmpty()) { + return new ArrayList<>(); + } + if (patterns.size() == 1) { + return patterns; } - // 3. If only "source.pattern" is specified, parse it using the helper method. - if (pattern != null) { - return buildUnionPattern( - isTreeModelDataAllowedToBeCaptured, - parsePatternsFromPatternParameter( - pattern, sourceParameters, isTreeModelDataAllowedToBeCaptured)); + final List<TreePattern> optimized = new ArrayList<>(); + // Determine coverage using base paths + for (int i = 0; i < patterns.size(); i++) { + final TreePattern current = patterns.get(i); + boolean isCoveredByOther = false; + + for (int j = 0; j < patterns.size(); j++) { + if (i == j) { + continue; + } + final TreePattern other = patterns.get(j); + + // If 'other' covers 'current', 'current' is redundant. + // Note: if they mutually cover each other (duplicates), we must ensure we keep one. + // We use index comparison to break ties for exact duplicates. + if (covers(other, current)) { + if (covers(current, other)) { + // Both cover each other (likely identical). Keep the one with smaller index. + if (j < i) { + isCoveredByOther = true; + break; + } + } else { + // Strict coverage + isCoveredByOther = true; + break; + } + } + } + + if (!isCoveredByOther) { + optimized.add(current); + } + } + return optimized; + } Review Comment: Referring to PathPatternTree, would it be faster to construct a tree and conclude from it？ -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
