keith-turner commented on a change in pull request #1873: URL: https://github.com/apache/accumulo/pull/1873#discussion_r561233283
########## File path: core/src/test/java/org/apache/accumulo/core/conf/DeprecatedPropertyUtilTest.java ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.conf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; + +import org.apache.commons.configuration2.CompositeConfiguration; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DeprecatedPropertyUtilTest { Review comment: Seems like it would be useful to also have some test in this class that rename master props to manager props. ########## File path: core/src/test/java/org/apache/accumulo/core/conf/DeprecatedPropertyUtilTest.java ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.conf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; + +import org.apache.commons.configuration2.CompositeConfiguration; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DeprecatedPropertyUtilTest { + private static class TestPropertyUtil extends DeprecatedPropertyUtil { + public static void registerTestRenamer() { + renamers.add(new PropertyRenamer() { + private static final String OLD_PREFIX = "old."; + + @Override + public boolean matches(String property) { + return property.startsWith(OLD_PREFIX); + } + + @Override + public String rename(String property) { + return "new." + property.substring(OLD_PREFIX.length()); + } + }); + } + } + + @BeforeClass + public static void setup() { + TestPropertyUtil.registerTestRenamer(); + } + + @Test + public void testNonDeprecatedPropertyRename() { + String oldProp = "some_property_name"; + String newProp = DeprecatedPropertyUtil.renameDeprecatedProperty(oldProp); + assertEquals(oldProp, newProp); + assertSame(oldProp, newProp); + } + + @Test + public void testDeprecatedPropertyRename() { + String newProp = DeprecatedPropertyUtil.renameDeprecatedProperty("old.test", false); + assertEquals("new.test", newProp); + } + + @Test + public void testSanityCheckWithOldProp() { + CompositeConfiguration config = new CompositeConfiguration(); + config.setProperty("old.prop", "3"); + DeprecatedPropertyUtil.sanityCheck(config); + } + + @Test + public void testSanityCheckWithNewProp() { + CompositeConfiguration config = new CompositeConfiguration(); + config.setProperty("new.prop", "4"); + DeprecatedPropertyUtil.sanityCheck(config); + } + Review comment: Could also have a sanity check test w/ new and old prefix that do not resolve to same, like "old.prop1" and "new.prop2". ########## File path: server/manager/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java ########## @@ -201,6 +204,39 @@ private void upgradeRootTabletMetadata(ServerContext ctx) { delete(ctx, ZROOT_TABLET_PATH); } + @SuppressWarnings("removal") + @Deprecated(since = "2.1.0", forRemoval = true) + private void renameMasterProps(ServerContext ctx) { + try { + // Figure out which props with the "master." prefix are set only in Zookeeper. If + // a property exists in the config and is either not set in the site config, or has + // a different value from the site config, then the property is set in zookeeper + // and we want to upgrade. + HashMap<String,String> configProps = new HashMap<>(); + HashMap<String,String> siteProps = new HashMap<>(); + ctx.getConfiguration().getProperties(configProps, + p -> p.startsWith(DeprecatedPropertyUtil.MasterPropertyRenamer.MASTER_PREFIX)); + ctx.getSiteConfiguration().getProperties(siteProps, + p -> p.startsWith(DeprecatedPropertyUtil.MasterPropertyRenamer.MASTER_PREFIX)); + configProps.entrySet().removeIf( + e -> siteProps.containsKey(e.getKey()) && e.getValue().equals(siteProps.get(e.getKey()))); Review comment: Seems like if `master.prop1=v1` is set in site and ZK, then it will not be renamed in ZK? If so, why not rename it in ZK? ########## File path: server/manager/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java ########## @@ -201,6 +204,39 @@ private void upgradeRootTabletMetadata(ServerContext ctx) { delete(ctx, ZROOT_TABLET_PATH); } + @SuppressWarnings("removal") + @Deprecated(since = "2.1.0", forRemoval = true) + private void renameMasterProps(ServerContext ctx) { + try { + // Figure out which props with the "master." prefix are set only in Zookeeper. If + // a property exists in the config and is either not set in the site config, or has + // a different value from the site config, then the property is set in zookeeper + // and we want to upgrade. + HashMap<String,String> configProps = new HashMap<>(); + HashMap<String,String> siteProps = new HashMap<>(); + ctx.getConfiguration().getProperties(configProps, + p -> p.startsWith(DeprecatedPropertyUtil.MasterPropertyRenamer.MASTER_PREFIX)); + ctx.getSiteConfiguration().getProperties(siteProps, + p -> p.startsWith(DeprecatedPropertyUtil.MasterPropertyRenamer.MASTER_PREFIX)); + configProps.entrySet().removeIf( + e -> siteProps.containsKey(e.getKey()) && e.getValue().equals(siteProps.get(e.getKey()))); + + for (Entry<String,String> entry : configProps.entrySet()) { + // Set the property under the new name + SystemPropUtil.setSystemProperty(ctx, + DeprecatedPropertyUtil.renameDeprecatedProperty(entry.getKey(), false), Review comment: Seems `DeprecatedPropertyUtil.renameDeprecatedProperty` is called explicitly here to avoid logging. That is nice because the log message does not fit this situation. Made me think a log message geared towards upgrade may be useful. Like `During upgrade property X was renamed to Y` ########## File path: core/src/test/java/org/apache/accumulo/core/conf/DeprecatedPropertyUtilTest.java ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.conf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; + +import org.apache.commons.configuration2.CompositeConfiguration; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DeprecatedPropertyUtilTest { + private static class TestPropertyUtil extends DeprecatedPropertyUtil { + public static void registerTestRenamer() { + renamers.add(new PropertyRenamer() { + private static final String OLD_PREFIX = "old."; + + @Override + public boolean matches(String property) { + return property.startsWith(OLD_PREFIX); + } + + @Override + public String rename(String property) { + return "new." + property.substring(OLD_PREFIX.length()); + } + }); + } + } + + @BeforeClass + public static void setup() { + TestPropertyUtil.registerTestRenamer(); + } + + @Test + public void testNonDeprecatedPropertyRename() { + String oldProp = "some_property_name"; + String newProp = DeprecatedPropertyUtil.renameDeprecatedProperty(oldProp); + assertEquals(oldProp, newProp); + assertSame(oldProp, newProp); + } + + @Test + public void testDeprecatedPropertyRename() { + String newProp = DeprecatedPropertyUtil.renameDeprecatedProperty("old.test", false); + assertEquals("new.test", newProp); + } + + @Test + public void testSanityCheckWithOldProp() { + CompositeConfiguration config = new CompositeConfiguration(); + config.setProperty("old.prop", "3"); + DeprecatedPropertyUtil.sanityCheck(config); + } + + @Test + public void testSanityCheckWithNewProp() { + CompositeConfiguration config = new CompositeConfiguration(); + config.setProperty("new.prop", "4"); + DeprecatedPropertyUtil.sanityCheck(config); + } + + @Test + public void testSanityCheckFailure() { + CompositeConfiguration config = new CompositeConfiguration(); + config.setProperty("old.prop", "3"); + config.setProperty("new.prop", "4"); + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> DeprecatedPropertyUtil.sanityCheck(config)); + assertEquals("new.prop and deprecated old.prop cannot both be site in the configuration.", Review comment: `cannot both be site in the` does not seem like what you intended to write. ########## File path: core/src/test/java/org/apache/accumulo/core/conf/DeprecatedPropertyUtilTest.java ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.conf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; + +import org.apache.commons.configuration2.CompositeConfiguration; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DeprecatedPropertyUtilTest { Review comment: > I was torn about that since the master to manager prop renaming is deprecated, and would therefore eventually need to be removed from this test. That makes sense. It would be nice to test what the code that users will run. When its removed in the future, if the test fails cleanly in such a way that the cause is obvious, I think it will be easy for someone to clean up the test. ########## File path: core/src/test/java/org/apache/accumulo/core/conf/DeprecatedPropertyUtilTest.java ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.conf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; + +import org.apache.commons.configuration2.CompositeConfiguration; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DeprecatedPropertyUtilTest { Review comment: > I was torn about that since the master to manager prop renaming is deprecated, and would therefore eventually need to be removed from this test. That makes sense. It would be nice to test the code that users will run. When its removed in the future, if the test fails cleanly in such a way that the cause is obvious, I think it will be easy for someone to clean up the test. ########## File path: server/manager/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java ########## @@ -201,6 +204,39 @@ private void upgradeRootTabletMetadata(ServerContext ctx) { delete(ctx, ZROOT_TABLET_PATH); } + @SuppressWarnings("removal") + @Deprecated(since = "2.1.0", forRemoval = true) + private void renameMasterProps(ServerContext ctx) { + try { + // Figure out which props with the "master." prefix are set only in Zookeeper. If + // a property exists in the config and is either not set in the site config, or has + // a different value from the site config, then the property is set in zookeeper + // and we want to upgrade. + HashMap<String,String> configProps = new HashMap<>(); + HashMap<String,String> siteProps = new HashMap<>(); + ctx.getConfiguration().getProperties(configProps, + p -> p.startsWith(DeprecatedPropertyUtil.MasterPropertyRenamer.MASTER_PREFIX)); + ctx.getSiteConfiguration().getProperties(siteProps, + p -> p.startsWith(DeprecatedPropertyUtil.MasterPropertyRenamer.MASTER_PREFIX)); + configProps.entrySet().removeIf( + e -> siteProps.containsKey(e.getKey()) && e.getValue().equals(siteProps.get(e.getKey()))); Review comment: I missed that configProps was a union of the site config and ZK config, I see what you are trying to do now. > Is there another utility for retrieving ZK properties while ignoring the contents of the site file Looking around in the code, one way might be to create a ZooConfiguration object directly with an empty parent. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
