brianloss commented on a change in pull request #1873: URL: https://github.com/apache/accumulo/pull/1873#discussion_r561261206
########## File path: core/src/test/java/org/apache/accumulo/core/conf/DeprecatedPropertyUtilTest.java ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.conf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; + +import org.apache.commons.configuration2.CompositeConfiguration; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DeprecatedPropertyUtilTest { Review comment: I was torn about that since the master to manager prop renaming is deprecated, and would therefore eventually need to be removed from this test. I was thinking what I added was sufficient to test the general functionality of the class, but am happy to add master to manager rename checks as well if you think it's worth it even though they'll be removed eventually. ########## File path: core/src/test/java/org/apache/accumulo/core/conf/DeprecatedPropertyUtilTest.java ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.conf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; + +import org.apache.commons.configuration2.CompositeConfiguration; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DeprecatedPropertyUtilTest { + private static class TestPropertyUtil extends DeprecatedPropertyUtil { + public static void registerTestRenamer() { + renamers.add(new PropertyRenamer() { + private static final String OLD_PREFIX = "old."; + + @Override + public boolean matches(String property) { + return property.startsWith(OLD_PREFIX); + } + + @Override + public String rename(String property) { + return "new." + property.substring(OLD_PREFIX.length()); + } + }); + } + } + + @BeforeClass + public static void setup() { + TestPropertyUtil.registerTestRenamer(); + } + + @Test + public void testNonDeprecatedPropertyRename() { + String oldProp = "some_property_name"; + String newProp = DeprecatedPropertyUtil.renameDeprecatedProperty(oldProp); + assertEquals(oldProp, newProp); + assertSame(oldProp, newProp); + } + + @Test + public void testDeprecatedPropertyRename() { + String newProp = DeprecatedPropertyUtil.renameDeprecatedProperty("old.test", false); + assertEquals("new.test", newProp); + } + + @Test + public void testSanityCheckWithOldProp() { + CompositeConfiguration config = new CompositeConfiguration(); + config.setProperty("old.prop", "3"); + DeprecatedPropertyUtil.sanityCheck(config); + } + + @Test + public void testSanityCheckWithNewProp() { + CompositeConfiguration config = new CompositeConfiguration(); + config.setProperty("new.prop", "4"); + DeprecatedPropertyUtil.sanityCheck(config); + } + + @Test + public void testSanityCheckFailure() { + CompositeConfiguration config = new CompositeConfiguration(); + config.setProperty("old.prop", "3"); + config.setProperty("new.prop", "4"); + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> DeprecatedPropertyUtil.sanityCheck(config)); + assertEquals("new.prop and deprecated old.prop cannot both be site in the configuration.", Review comment: Oops, good catch. That was supposed to be set not site. Fixed in the next commit. ########## File path: server/manager/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java ########## @@ -201,6 +204,39 @@ private void upgradeRootTabletMetadata(ServerContext ctx) { delete(ctx, ZROOT_TABLET_PATH); } + @SuppressWarnings("removal") + @Deprecated(since = "2.1.0", forRemoval = true) + private void renameMasterProps(ServerContext ctx) { + try { + // Figure out which props with the "master." prefix are set only in Zookeeper. If + // a property exists in the config and is either not set in the site config, or has + // a different value from the site config, then the property is set in zookeeper + // and we want to upgrade. + HashMap<String,String> configProps = new HashMap<>(); + HashMap<String,String> siteProps = new HashMap<>(); + ctx.getConfiguration().getProperties(configProps, + p -> p.startsWith(DeprecatedPropertyUtil.MasterPropertyRenamer.MASTER_PREFIX)); + ctx.getSiteConfiguration().getProperties(siteProps, + p -> p.startsWith(DeprecatedPropertyUtil.MasterPropertyRenamer.MASTER_PREFIX)); + configProps.entrySet().removeIf( + e -> siteProps.containsKey(e.getKey()) && e.getValue().equals(siteProps.get(e.getKey()))); Review comment: I was trying to list only properties that are set in ZK. I couldn't find a better way to determine that, and modeled this after code in [ConfigCommand](https://github.com/apache/accumulo/blob/main/shell/src/main/java/org/apache/accumulo/shell/commands/ConfigCommand.java). In the case you mention, the ZK property is superfluous and really should be removed rather than renamed if it matches what's in site already, but I'm not doing that either. Is there another utility for retrieving ZK properties while ignoring the contents of the site file, command-line and env properties? Maybe I need to write something if not. ########## File path: core/src/test/java/org/apache/accumulo/core/conf/DeprecatedPropertyUtilTest.java ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.conf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; + +import org.apache.commons.configuration2.CompositeConfiguration; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DeprecatedPropertyUtilTest { + private static class TestPropertyUtil extends DeprecatedPropertyUtil { + public static void registerTestRenamer() { + renamers.add(new PropertyRenamer() { + private static final String OLD_PREFIX = "old."; + + @Override + public boolean matches(String property) { + return property.startsWith(OLD_PREFIX); + } + + @Override + public String rename(String property) { + return "new." + property.substring(OLD_PREFIX.length()); + } + }); + } + } + + @BeforeClass + public static void setup() { + TestPropertyUtil.registerTestRenamer(); + } + + @Test + public void testNonDeprecatedPropertyRename() { + String oldProp = "some_property_name"; + String newProp = DeprecatedPropertyUtil.renameDeprecatedProperty(oldProp); + assertEquals(oldProp, newProp); + assertSame(oldProp, newProp); + } + + @Test + public void testDeprecatedPropertyRename() { + String newProp = DeprecatedPropertyUtil.renameDeprecatedProperty("old.test", false); + assertEquals("new.test", newProp); + } + + @Test + public void testSanityCheckWithOldProp() { + CompositeConfiguration config = new CompositeConfiguration(); + config.setProperty("old.prop", "3"); + DeprecatedPropertyUtil.sanityCheck(config); + } + + @Test + public void testSanityCheckWithNewProp() { + CompositeConfiguration config = new CompositeConfiguration(); + config.setProperty("new.prop", "4"); + DeprecatedPropertyUtil.sanityCheck(config); + } + Review comment: Added in next commit. ########## File path: server/manager/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java ########## @@ -201,6 +204,39 @@ private void upgradeRootTabletMetadata(ServerContext ctx) { delete(ctx, ZROOT_TABLET_PATH); } + @SuppressWarnings("removal") + @Deprecated(since = "2.1.0", forRemoval = true) + private void renameMasterProps(ServerContext ctx) { + try { + // Figure out which props with the "master." prefix are set only in Zookeeper. If + // a property exists in the config and is either not set in the site config, or has + // a different value from the site config, then the property is set in zookeeper + // and we want to upgrade. + HashMap<String,String> configProps = new HashMap<>(); + HashMap<String,String> siteProps = new HashMap<>(); + ctx.getConfiguration().getProperties(configProps, + p -> p.startsWith(DeprecatedPropertyUtil.MasterPropertyRenamer.MASTER_PREFIX)); + ctx.getSiteConfiguration().getProperties(siteProps, + p -> p.startsWith(DeprecatedPropertyUtil.MasterPropertyRenamer.MASTER_PREFIX)); + configProps.entrySet().removeIf( + e -> siteProps.containsKey(e.getKey()) && e.getValue().equals(siteProps.get(e.getKey()))); + + for (Entry<String,String> entry : configProps.entrySet()) { + // Set the property under the new name + SystemPropUtil.setSystemProperty(ctx, + DeprecatedPropertyUtil.renameDeprecatedProperty(entry.getKey(), false), Review comment: Sure, good idea. I'll add in the next commit. ########## File path: core/src/test/java/org/apache/accumulo/core/conf/DeprecatedPropertyUtilTest.java ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.conf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; + +import org.apache.commons.configuration2.CompositeConfiguration; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DeprecatedPropertyUtilTest { Review comment: Ok, added. ########## File path: server/manager/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java ########## @@ -201,6 +204,39 @@ private void upgradeRootTabletMetadata(ServerContext ctx) { delete(ctx, ZROOT_TABLET_PATH); } + @SuppressWarnings("removal") + @Deprecated(since = "2.1.0", forRemoval = true) + private void renameMasterProps(ServerContext ctx) { + try { + // Figure out which props with the "master." prefix are set only in Zookeeper. If + // a property exists in the config and is either not set in the site config, or has + // a different value from the site config, then the property is set in zookeeper + // and we want to upgrade. + HashMap<String,String> configProps = new HashMap<>(); + HashMap<String,String> siteProps = new HashMap<>(); + ctx.getConfiguration().getProperties(configProps, + p -> p.startsWith(DeprecatedPropertyUtil.MasterPropertyRenamer.MASTER_PREFIX)); + ctx.getSiteConfiguration().getProperties(siteProps, + p -> p.startsWith(DeprecatedPropertyUtil.MasterPropertyRenamer.MASTER_PREFIX)); + configProps.entrySet().removeIf( + e -> siteProps.containsKey(e.getKey()) && e.getValue().equals(siteProps.get(e.getKey()))); Review comment: I tried that, and it looks a little cleaner (and catches the edge case). In the next commit... ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
