This is an automated email from the ASF dual-hosted git repository. ramackri pushed a commit to branch ranger-kafka-dynamic-partition-plan in repository https://gitbox.apache.org/repos/asf/ranger.git
commit da27821c25ca2661023e8599dd874aa45a0aeaa5 Author: ramk <[email protected]> AuthorDate: Sun Jun 21 13:01:51 2026 +0530 Apply auth_to_local startup fallback and address code review feedback. When dynamic partition-plan mode is enabled but the registry topic does not exist yet, fall back to the full XML auth_to_local catalog until bootstrap. Rename composer helpers for clarity, remove dead catalog code, and fix partition-plan E2E scripts for dynamic mode toggling and auth access tests. Co-authored-by: Cursor <[email protected]> --- .../ranger/audit/utils/AuditMessageQueueUtils.java | 19 ++++ .../kafka/partition/AuthToLocalRuleCatalog.java | 17 +--- .../kafka/partition/AuthToLocalRuleComposer.java | 48 +++++++-- .../kafka/partition/PartitionPlanService.java | 2 +- .../org/apache/ranger/audit/rest/AuditREST.java | 9 +- .../partition/AuthToLocalRuleComposerTest.java | 112 +++++++++++++++++---- .../scripts/audit/partition-plan-e2e-lib.sh | 40 ++++++-- .../scripts/audit/verify-partition-plan-e2e-all.sh | 12 ++- .../scripts/audit/verify-partition-plan-e2e.sh | 2 +- 9 files changed, 207 insertions(+), 54 deletions(-) diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java index 29770e9e2..f1e831257 100644 --- a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java +++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/utils/AuditMessageQueueUtils.java @@ -177,6 +177,25 @@ public static String createPartitionPlanTopicIfNotExists(Properties props, Strin throw new RuntimeException("Failed to create partition plan topic '" + planTopic + "'"); } + /** + * Returns whether the compacted partition-plan registry topic already exists on the audit Kafka cluster. + * When the check fails (broker unreachable, ACL denied), returns {@code false} so callers can fall back + * to static XML {@code auth_to_local} rules until the plan topic is available. + */ + public static boolean partitionPlanTopicExists(Properties props, String propPrefix) { + String planTopic = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_PARTITION_PLAN_TOPIC, AuditServerConstants.DEFAULT_PARTITION_PLAN_TOPIC); + Map<String, Object> adminConfig = buildAdminClientConfig(props, propPrefix); + try (AdminClient admin = AdminClient.create(adminConfig)) { + Set<String> topicNames = admin.listTopics().names().get(); + boolean exists = topicNames.contains(planTopic); + LOG.debug("Partition plan topic '{}' exists: {}", planTopic, exists); + return exists; + } catch (Exception ex) { + LOG.warn("Could not determine whether partition plan topic '{}' exists: {}. Assuming it does not.", planTopic, ex.getMessage()); + return false; + } + } + public static Map<String, Object> buildAdminClientConfig(Properties props, String propPrefix) { String bootstrapServers = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_BOOTSTRAP_SERVERS); String securityProtocol = MiscUtil.getStringProperty(props, propPrefix + "." + AuditServerConstants.PROP_SECURITY_PROTOCOL, AuditServerConstants.DEFAULT_SECURITY_PROTOCOL); diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleCatalog.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleCatalog.java index f11be02f2..35ecb87c1 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleCatalog.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleCatalog.java @@ -24,10 +24,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -86,9 +84,8 @@ String compose(Set<String> activeShortNames) { return composeFull(); } - List<String> lines = new ArrayList<>(); - Set<String> covered = new LinkedHashSet<>(); - Map<String, String> catalogByShortName = catalogRulesByShortName(); + List<String> lines = new ArrayList<>(); + Set<String> covered = new LinkedHashSet<>(); for (CatalogEntry entry : primaryRulesInOrder) { if (entry.targetShortName != null && active.contains(entry.targetShortName)) { @@ -107,16 +104,6 @@ String compose(Set<String> activeShortNames) { return joinRules(lines); } - private Map<String, String> catalogRulesByShortName() { - Map<String, String> ret = new LinkedHashMap<>(); - for (CatalogEntry entry : primaryRulesInOrder) { - if (entry.targetShortName != null) { - ret.putIfAbsent(entry.targetShortName, entry.ruleLine); - } - } - return ret; - } - private static List<String> tokenizeRules(String rawRules) { if (StringUtils.isBlank(rawRules)) { return Collections.emptyList(); diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java index 59ad9511b..97eda9fef 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposer.java @@ -25,6 +25,7 @@ import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; import org.apache.ranger.audit.server.AuditServerConfig; import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.audit.utils.AuditMessageQueueUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +46,7 @@ public final class AuthToLocalRuleComposer { private volatile AuthToLocalRuleCatalog catalog; private volatile String lastAppliedRules; private volatile int lastAppliedPlanVersion; + private volatile Boolean planTopicExistsOverrideForTests; private AuthToLocalRuleComposer() { } @@ -73,6 +75,24 @@ public void applyStaticRules() { applyRules(loaded.composeFull(), 0); } + /** + * Dynamic-mode startup: when the partition-plan Kafka topic does not exist yet, apply the full XML + * catalog so Kerberos mapping works before {@link PartitionPlanWatcher} bootstraps the registry. + * When the topic already exists, defer to composed rules on {@link PartitionPlanHolder#install}. + */ + public void applyStartupRulesForDynamicMode(Properties props, String ingestorPropPrefix) { + if (!PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(props, ingestorPropPrefix)) { + return; + } + requireCatalog(); + if (isPlanTopicPresent(props, ingestorPropPrefix)) { + LOG.info("Partition plan topic exists; auth_to_local rules will be composed from allowlisted short names on plan install"); + } else { + applyStaticRules(); + LOG.info("Partition plan topic does not exist yet; applied full auth_to_local catalog from XML until plan bootstrap"); + } + } + /** * When dynamic partition-plan mode is enabled, compose rules from the union of allowlisted short * names and install them before audit REST authorization runs. @@ -87,24 +107,38 @@ public void applyForPlan(PartitionPlan plan) { } AuthToLocalRuleCatalog loaded = requireCatalog(); - Set<String> active = unionAllowedUsers(plan); - String rules = loaded.compose(active); + Set<String> activeShortNames = collectAllowedUserShortNames(plan); + String rules = loaded.compose(activeShortNames); applyRules(rules, plan.getVersion()); - LOG.info("Applied composed auth_to_local rules for plan version {} ({} active short names)", plan.getVersion(), active.size()); + LOG.info("Applied composed auth_to_local rules for plan version {} ({} active short names)", plan.getVersion(), activeShortNames.size()); } /** Visible for tests — composes without applying Kerberos rules. */ - String composeForActiveShortNames(Set<String> activeShortNames) { + String composeRulesForShortNames(Set<String> activeShortNames) { return requireCatalog().compose(activeShortNames); } /** Clears cached apply state between unit tests. */ public synchronized void resetForTests() { - lastAppliedRules = null; - lastAppliedPlanVersion = 0; + lastAppliedRules = null; + lastAppliedPlanVersion = 0; + planTopicExistsOverrideForTests = null; + } + + /** When non-null, overrides {@link AuditMessageQueueUtils#partitionPlanTopicExists} for unit tests. */ + void setPlanTopicExistsOverrideForTests(Boolean exists) { + planTopicExistsOverrideForTests = exists; + } + + private boolean isPlanTopicPresent(Properties props, String ingestorPropPrefix) { + Boolean override = planTopicExistsOverrideForTests; + if (override != null) { + return override; + } + return AuditMessageQueueUtils.partitionPlanTopicExists(props, ingestorPropPrefix); } - static Set<String> unionAllowedUsers(PartitionPlan plan) { + static Set<String> collectAllowedUserShortNames(PartitionPlan plan) { Set<String> active = new LinkedHashSet<>(); if (plan == null || plan.getServices() == null) { return active; diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java index 74760ee67..ca1a1c7f1 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/producer/kafka/partition/PartitionPlanService.java @@ -37,7 +37,7 @@ /** REST mutations and reads for the dynamic Kafka partition plan. */ @Component public class PartitionPlanService { - static final String INGESTOR_PROP_PREFIX = "ranger.audit.ingestor"; + public static final String INGESTOR_PROP_PREFIX = "ranger.audit.ingestor"; private final Properties configProps; private final PartitionPlanHolder holder; diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index 1a2c223b7..73b533cfb 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -536,8 +536,9 @@ private String applyAuthToLocal(String principal) { /** * Loads the auth_to_local catalog from ranger-audit-ingestor-site.xml. When dynamic partition-plan - * mode is disabled, applies the full catalog immediately. When enabled, rules are composed from - * partition-plan allowlists on {@link PartitionPlanHolder#install(PartitionPlan, Integer)}. + * mode is disabled, applies the full catalog immediately. When enabled, applies static XML rules + * only if the partition-plan Kafka topic does not exist yet; otherwise composed rules are installed + * on {@link PartitionPlanHolder#install(PartitionPlan, Integer)} via {@link AuthToLocalRuleComposer#applyForPlan}. */ private static void initializeAuthToLocal() { AuditServerConfig config = AuditServerConfig.getInstance(); @@ -550,8 +551,8 @@ private static void initializeAuthToLocal() { AuthToLocalRuleComposer composer = AuthToLocalRuleComposer.getInstance(); composer.initializeFromConfig(); - if (PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(config.getProperties(), AuditServerConstants.PROP_PREFIX_AUDIT_SERVER.substring(0, AuditServerConstants.PROP_PREFIX_AUDIT_SERVER.length() - 1))) { - LOG.info("Dynamic partition plan enabled; auth_to_local rules will be composed from allowlisted short names on plan install"); + if (PartitionPlanKafkaConfig.isDynamicPartitionPlanEnabled(config.getProperties(), PartitionPlanService.INGESTOR_PROP_PREFIX)) { + composer.applyStartupRulesForDynamicMode(config.getProperties(), PartitionPlanService.INGESTOR_PROP_PREFIX); } else { composer.applyStaticRules(); LOG.debug("Applied static auth_to_local catalog from site XML"); diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposerTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposerTest.java index 01b2bbc32..1cff02d6f 100644 --- a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposerTest.java +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/producer/kafka/partition/AuthToLocalRuleComposerTest.java @@ -1,18 +1,20 @@ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.ranger.audit.producer.kafka.partition; @@ -21,6 +23,7 @@ import org.apache.ranger.audit.producer.kafka.partition.model.PartitionPlan; import org.apache.ranger.audit.producer.kafka.partition.model.PluginPartitionAssignment; import org.apache.ranger.audit.producer.kafka.partition.model.ServiceAllowlistEntry; +import org.apache.ranger.audit.server.AuditServerConfig; import org.apache.ranger.audit.server.AuditServerConstants; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -33,6 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; public class AuthToLocalRuleComposerTest { @@ -47,6 +51,9 @@ public class AuthToLocalRuleComposerTest { + "RULE:[1:$1@$0](.*@.*)s/@.*// " + "DEFAULT"; + private static final String SENTINEL_NN_RULE = + "RULE:[2:$1/$2@$0](nn/.*@.*)s/.*/startup-not-applied/ DEFAULT"; + private AuthToLocalRuleComposer composer; @BeforeEach @@ -62,11 +69,14 @@ public void setUp() { public void tearDown() { composer.resetForTests(); PartitionPlanHolder.getInstance().resetForTests(); + AuditServerConfig.getInstance().set( + PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, + "false"); } @Test public void testComposeKmsOnlyIncludesRangerkmsRuleAndTail() { - String rules = composer.composeForActiveShortNames(Set.of("rangerkms")); + String rules = composer.composeRulesForShortNames(Set.of("rangerkms")); assertTrue(rules.contains("(rangerkms/.*@.*)s/.*/rangerkms/")); assertFalse(rules.contains("(hive/.*@.*)s/.*/hive/")); @@ -76,7 +86,7 @@ public void testComposeKmsOnlyIncludesRangerkmsRuleAndTail() { @Test public void testComposeHdfsUsesCatalogRegexNotSimpleRule() { - String rules = composer.composeForActiveShortNames(Set.of("hdfs")); + String rules = composer.composeRulesForShortNames(Set.of("hdfs")); assertTrue(rules.contains("([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/")); assertFalse(rules.contains("(hdfs/.*@.*)s/.*/hdfs/")); @@ -84,7 +94,7 @@ public void testComposeHdfsUsesCatalogRegexNotSimpleRule() { @Test public void testComposeOzoneIncludesMultipleShortNames() { - String rules = composer.composeForActiveShortNames(Set.of("ozone", "om", "scm", "dn")); + String rules = composer.composeRulesForShortNames(Set.of("ozone", "om", "scm", "dn")); assertTrue(rules.contains("(ozone/.*@.*)s/.*/ozone/")); assertTrue(rules.contains("(om/.*@.*)s/.*/om/")); @@ -92,13 +102,13 @@ public void testComposeOzoneIncludesMultipleShortNames() { @Test public void testComposeUnknownShortNameGeneratesSimpleRule() { - String rules = composer.composeForActiveShortNames(Set.of("myapp")); + String rules = composer.composeRulesForShortNames(Set.of("myapp")); assertTrue(rules.contains("(myapp/.*@.*)s/.*/myapp/")); } @Test - public void testUnionAllowedUsersFromPlan() { + public void testCollectAllowedUserShortNamesFromPlan() { Map<String, ServiceAllowlistEntry> services = new LinkedHashMap<>(); services.put("dev_kms", ServiceAllowlistEntry.ofUsers("rangerkms")); services.put("dev_ozone", ServiceAllowlistEntry.ofUsers("om", "ozone")); @@ -111,14 +121,14 @@ public void testUnionAllowedUsersFromPlan() { .services(services) .build(); - Set<String> active = AuthToLocalRuleComposer.unionAllowedUsers(plan); + Set<String> activeShortNames = AuthToLocalRuleComposer.collectAllowedUserShortNames(plan); - assertEquals(Set.of("rangerkms", "om", "ozone"), active); + assertEquals(Set.of("rangerkms", "om", "ozone"), activeShortNames); } @Test public void testKerberosMappingForComposedKmsRules() throws Exception { - String rules = composer.composeForActiveShortNames(Set.of("rangerkms")); + String rules = composer.composeRulesForShortNames(Set.of("rangerkms")); KerberosName.setRules(rules); assertEquals("rangerkms", new KerberosName("rangerkms/[email protected]").getShortName()); @@ -126,12 +136,69 @@ public void testKerberosMappingForComposedKmsRules() throws Exception { @Test public void testKerberosMappingForComposedHdfsRules() throws Exception { - String rules = composer.composeForActiveShortNames(Set.of("hdfs")); + String rules = composer.composeRulesForShortNames(Set.of("hdfs")); KerberosName.setRules(rules); assertEquals("hdfs", new KerberosName("nn/[email protected]").getShortName()); } + @Test + public void testStartupDynamicWhenTopicMissingAppliesStaticCatalog() throws Exception { + KerberosName.setRules(SENTINEL_NN_RULE); + Properties props = dynamicModeProps(); + composer.setPlanTopicExistsOverrideForTests(false); + composer.applyStartupRulesForDynamicMode(props, PartitionPlanService.INGESTOR_PROP_PREFIX); + + assertEquals("hdfs", new KerberosName("nn/[email protected]").getShortName()); + } + + @Test + public void testStartupDynamicWhenTopicExistsDefersStaticCatalog() throws Exception { + KerberosName.setRules(SENTINEL_NN_RULE); + Properties props = dynamicModeProps(); + composer.setPlanTopicExistsOverrideForTests(true); + composer.applyStartupRulesForDynamicMode(props, PartitionPlanService.INGESTOR_PROP_PREFIX); + + assertEquals("startup-not-applied", new KerberosName("nn/[email protected]").getShortName()); + } + + @Test + public void testStartupDynamicWhenDisabledIsNoOp() throws Exception { + KerberosName.setRules(SENTINEL_NN_RULE); + Properties props = new Properties(); + props.setProperty(AuditServerConstants.PROP_AUTH_TO_LOCAL, SAMPLE_CATALOG); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "false"); + composer.setPlanTopicExistsOverrideForTests(false); + composer.applyStartupRulesForDynamicMode(props, PartitionPlanService.INGESTOR_PROP_PREFIX); + + assertEquals("startup-not-applied", new KerberosName("nn/[email protected]").getShortName()); + } + + @Test + public void testInstallPlanAppliesComposedAuthToLocalRules() throws Exception { + AuditServerConfig config = AuditServerConfig.getInstance(); + config.set(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + config.set(AuditServerConstants.PROP_AUTH_TO_LOCAL, SAMPLE_CATALOG); + composer.initializeFromConfig(); + + Map<String, ServiceAllowlistEntry> services = new LinkedHashMap<>(); + services.put("dev_kms", ServiceAllowlistEntry.ofUsers("rangerkms")); + PartitionPlan plan = PartitionPlan.builder() + .topic("ranger_audits") + .version(3) + .topicPartitionCount(6) + .plugins(Map.of("kms", PluginPartitionAssignment.of(0, 1))) + .buffer(PluginPartitionAssignment.of(2, 3, 4, 5)) + .services(services) + .build(); + + PartitionPlanHolder.getInstance().install(plan, 6); + + assertEquals("rangerkms", new KerberosName("rangerkms/[email protected]").getShortName()); + assertThrows(KerberosName.NoMatchingRule.class, + () -> new KerberosName("nn/[email protected]").getShortName()); + } + @Test public void testExtractTargetShortNameFromCatalogLines() { assertEquals("hdfs", AuthToLocalRuleCatalog.extractTargetShortName( @@ -141,4 +208,11 @@ public void testExtractTargetShortNameFromCatalogLines() { assertEquals("mapred", AuthToLocalRuleCatalog.extractTargetShortName( "RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/")); } + + private static Properties dynamicModeProps() { + Properties props = new Properties(); + props.setProperty(AuditServerConstants.PROP_AUTH_TO_LOCAL, SAMPLE_CATALOG); + props.setProperty(PartitionPlanService.INGESTOR_PROP_PREFIX + "." + AuditServerConstants.PROP_PARTITION_PLAN_DYNAMIC_ENABLED, "true"); + return props; + } } diff --git a/dev-support/ranger-docker/scripts/audit/partition-plan-e2e-lib.sh b/dev-support/ranger-docker/scripts/audit/partition-plan-e2e-lib.sh index 97407e8b7..45927e089 100755 --- a/dev-support/ranger-docker/scripts/audit/partition-plan-e2e-lib.sh +++ b/dev-support/ranger-docker/scripts/audit/partition-plan-e2e-lib.sh @@ -194,6 +194,38 @@ pp_dynamic_enabled() { docker exec "${container}" grep -A1 "${PROP_DYNAMIC}" "${path}" 2>/dev/null | grep -qi '<value>true</value>' } +pp_patch_site_prop() { + local container="$1" + local site="$2" + local prop="$3" + local value="$4" + docker exec -u root "${container}" python3 - "${site}" "${prop}" "${value}" <<'PY' +import sys +import xml.etree.ElementTree as ET + +path, prop, value = sys.argv[1:4] +root = ET.parse(path).getroot() +found = False +for p in root.findall("property"): + name = p.find("name") + if name is None or (name.text or "").strip() != prop: + continue + val = p.find("value") + if val is None: + val = ET.SubElement(p, "value") + val.text = value + found = True + break +if not found: + prop_el = ET.SubElement(root, "property") + ET.SubElement(prop_el, "name").text = prop + ET.SubElement(prop_el, "value").text = value +ET.indent(root, space=" ") +tree = ET.ElementTree(root) +tree.write(path, encoding="unicode", xml_declaration=False) +PY +} + pp_set_dynamic_enabled() { local container="$1" local value="$2" @@ -203,11 +235,7 @@ pp_set_dynamic_enabled() { if ! docker exec "${container}" test -f "${site}" 2>/dev/null; then continue fi - if docker exec "${container}" grep -q "${PROP_DYNAMIC}" "${site}" 2>/dev/null; then - docker exec "${container}" sed -i "/${PROP_DYNAMIC}/{n;s|<value>.*</value>|<value>${value}</value>|;}" "${site}" - else - docker exec "${container}" sed -i "s|</configuration>| <property>\n <name>${PROP_DYNAMIC}</name>\n <value>${value}</value>\n </property>\n</configuration>|" "${site}" - fi + pp_patch_site_prop "${container}" "${site}" "${PROP_DYNAMIC}" "${value}" changed=true done if [[ "${changed}" == "true" && "${restart}" == "true" ]]; then @@ -241,7 +269,7 @@ pp_wait_watcher() { if [[ "${HTTP_CODE}" == "200" ]]; then return 0 fi - if docker logs "${log_args[@]}" "${container}" 2>&1 | grep -q "Partition plan watcher ready"; then + if docker logs ${log_args+"${log_args[@]}"} "${container}" 2>&1 | grep -q "Partition plan watcher ready"; then return 0 fi sleep 5 diff --git a/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e-all.sh b/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e-all.sh index c06764f8b..46b7fa18e 100755 --- a/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e-all.sh +++ b/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e-all.sh @@ -20,6 +20,7 @@ # ./scripts/audit/verify-partition-plan-e2e-all.sh # ./scripts/audit/verify-partition-plan-e2e-all.sh --skip-kafka-down # ./scripts/audit/verify-partition-plan-e2e-all.sh --with-audit-smoke +# ./scripts/audit/verify-partition-plan-e2e-all.sh --with-auth-access set -euo pipefail @@ -27,12 +28,14 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "${SCRIPT_DIR}" SKIP_KAFKA_DOWN=false +WITH_AUTH_ACCESS=false EXTRA_ARGS=() while [[ $# -gt 0 ]]; do case "$1" in --skip-kafka-down) SKIP_KAFKA_DOWN=true; shift ;; --with-audit-smoke) EXTRA_ARGS+=(--with-audit-smoke); shift ;; + --with-auth-access) WITH_AUTH_ACCESS=true; shift ;; --timeout) EXTRA_ARGS+=(--timeout "${2:?}"); shift 2 ;; -h|--help) sed -n '19,23p' "$0" @@ -48,7 +51,9 @@ done chmod +x scripts/audit/verify-partition-plan-e2e.sh \ scripts/audit/verify-partition-plan-multipod-e2e.sh \ scripts/audit/verify-partition-plan-brownfield-e2e.sh \ - scripts/audit/verify-partition-plan-kafka-down-e2e.sh 2>/dev/null || true + scripts/audit/verify-partition-plan-kafka-down-e2e.sh \ + scripts/audit/verify-dynamic-auth-to-local-e2e.sh \ + scripts/audit/wait-for-audit-health.sh 2>/dev/null || true run_step() { local name="$1" @@ -80,5 +85,10 @@ if [[ "${SKIP_KAFKA_DOWN}" != "true" ]]; then ./scripts/audit/verify-partition-plan-kafka-down-e2e.sh fi +if [[ "${WITH_AUTH_ACCESS}" == "true" ]]; then + run_step "Dynamic auth_to_local + /access curl E2E" \ + ./scripts/audit/verify-dynamic-auth-to-local-e2e.sh --no-enable +fi + echo "" echo "All partition-plan E2E suites passed." diff --git a/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e.sh b/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e.sh index fc7c3668c..36f53ccd3 100755 --- a/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e.sh +++ b/dev-support/ranger-docker/scripts/audit/verify-partition-plan-e2e.sh @@ -173,7 +173,7 @@ run_dynamic_tests() { echo " Stale expectedVersion (409)..." pp_ingestor_request "${CONTAINER}" POST "${PLAN_URL}/promote" \ - "{\"pluginId\":\"${promote_plugin}\",\"partitionCount\":1,\"expectedVersion\":1}" + "{\"pluginId\":\"e2eStale409\",\"partitionCount\":2,\"expectedVersion\":1}" if [[ "${HTTP_CODE}" == "409" ]]; then pp_record_pass "stale expectedVersion -> 409" else
