bshashikant commented on a change in pull request #1959: URL: https://github.com/apache/ozone/pull/1959#discussion_r584377357
########## File path: hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DatanodeLayout.java ########## @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.debug; + +import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.ozone.container.common.impl.ContainerSet; +import org.apache.hadoop.ozone.container.common.volume.HddsVolume; +import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; +import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; + +import org.kohsuke.MetaInfServices; +import picocli.CommandLine.Spec; +import picocli.CommandLine.Model.CommandSpec; + +import picocli.CommandLine; + +import java.util.List; +import java.util.concurrent.Callable; + +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; + +/** + * Parse Ratis Log CLI implementation. + */ [email protected]( + name = "dnlayout", + description = "Shell of updating datanode layout format", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true) +@MetaInfServices(SubcommandWithParent.class) +public class DatanodeLayout extends GenericCli + implements Callable<Void>, SubcommandWithParent{ + + @CommandLine.Option(names = {"--path"}, + required = true, + description = "File Path") + private String storagePath; Review comment: We can make the path as an "optional" arg here. By default, the command can work against all storage locations. ########## File path: hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DatanodeLayout.java ########## @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.debug; + +import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.ozone.container.common.impl.ContainerSet; +import org.apache.hadoop.ozone.container.common.volume.HddsVolume; +import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; +import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; + +import org.kohsuke.MetaInfServices; +import picocli.CommandLine.Spec; +import picocli.CommandLine.Model.CommandSpec; + +import picocli.CommandLine; + +import java.util.List; +import java.util.concurrent.Callable; + +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; + +/** + * Parse Ratis Log CLI implementation. + */ [email protected]( + name = "dnlayout", + description = "Shell of updating datanode layout format", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true) +@MetaInfServices(SubcommandWithParent.class) +public class DatanodeLayout extends GenericCli + implements Callable<Void>, SubcommandWithParent{ + + @CommandLine.Option(names = {"--path"}, + required = true, + description = "File Path") + private String storagePath; + + @CommandLine.Option(names = {"--verify"}, + description = "Verify that the datanode layout is correct") + private boolean verify; + + @Spec + private CommandSpec spec; + + @Override + public Void call() throws Exception { + OzoneConfiguration conf = createOzoneConfiguration(); + + runUpgrade(conf, storagePath, verify); + return null; + } + + public static void main(String[] args) { + new DatanodeLayout().run(args); + } + + @Override + public Class<?> getParentType() { + return OzoneDebug.class; + } + + public static List<HddsVolume> runUpgrade(OzoneConfiguration conf, + String storagePath, boolean verify) throws Exception { + conf.unset(HDDS_DATANODE_DIR_KEY); + conf.set(HDDS_DATANODE_DIR_KEY, storagePath); + + if (verify) { + conf.setBoolean( + ScmConfigKeys.HDDS_DATANODE_UPGRADE_LAYOUT_INLINE, false); + } + + MutableVolumeSet volumeSet = new MutableVolumeSet(conf); + ContainerSet containerSet = new ContainerSet(); + OzoneContainer.buildContainerSet(volumeSet, containerSet, conf); Review comment: I think, we should decouple this logic from building the container set . We can run the tool to rename the locations and generate the required Yaml files and ensure containers can be loaded in the verify step. I am ok to get this patch in and do it in subsequent PR as well. ########## File path: hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java ########## @@ -171,19 +219,36 @@ private void verifyContainerFile(long containerID, File containerFile) { * @param containerData * @throws IOException */ - public void verifyAndFixupContainerData(ContainerData containerData) - throws IOException { + public void verifyAndFixupContainerData(File storageLoc, + ContainerData containerData) throws IOException { switch (containerData.getContainerType()) { case KeyValueContainer: if (containerData instanceof KeyValueContainerData) { KeyValueContainerData kvContainerData = (KeyValueContainerData) containerData; containerData.setVolume(hddsVolume); + KeyValueContainer kvContainer = null; + if (isInUpgradeMode) { + kvContainerData.setMetadataPath( + findNormalizedPath(storageLoc, + kvContainerData.getMetadataPath())); + kvContainerData.setChunksPath( + findNormalizedPath(storageLoc, + kvContainerData.getChunksPath())); - KeyValueContainerUtil.parseKVContainerData(kvContainerData, config); - KeyValueContainer kvContainer = new KeyValueContainer( - kvContainerData, config); + Yaml yaml = ContainerDataYaml.getYamlForContainerType( + containerData.getContainerType()); + containerData.computeAndSetChecksum(yaml); + KeyValueContainerUtil.parseKVContainerData(kvContainerData, config); Review comment: We can move the same steps here to another function. ########## File path: hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java ########## @@ -148,7 +167,36 @@ public boolean accept(File pathname) { LOG.info("Finish verifying containers on volume {}", hddsVolumeRootDir); } - private void verifyContainerFile(long containerID, File containerFile) { + public File preProcessStorageLoc(File storageLoc) throws Exception { + File clusterDir = getClusterDir(); + + if (!isInUpgradeMode) { + Preconditions.checkArgument(clusterDir.exists(), + "Storage Dir:" + clusterDir + " doesn't exists"); + Preconditions.checkArgument(storageLoc.equals(clusterDir), + "configured storage dir" + storageLoc + + " is not same as cluster UUID" + clusterDir); + return storageLoc; + } + Review comment: Can we change the msg to --> " Configured storage location path does not contain "clusterId" or something similar. ########## File path: hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java ########## @@ -198,4 +263,18 @@ public void verifyAndFixupContainerData(ContainerData containerData) ContainerProtos.Result.UNKNOWN_CONTAINER_TYPE); } } Review comment: Can we add some comments javadoc to describe what the function is supposed to do? ########## File path: hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DatanodeLayout.java ########## @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.debug; + +import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.ozone.container.common.impl.ContainerSet; +import org.apache.hadoop.ozone.container.common.volume.HddsVolume; +import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; +import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; + +import org.kohsuke.MetaInfServices; +import picocli.CommandLine.Spec; +import picocli.CommandLine.Model.CommandSpec; + +import picocli.CommandLine; + +import java.util.List; +import java.util.concurrent.Callable; + +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; + +/** + * Parse Ratis Log CLI implementation. Review comment: Misleading comment. ########## File path: hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java ########## @@ -83,19 +90,30 @@ this.containerSet = cset; this.config = conf; this.volumeSet = volSet; + this.isInUpgradeMode = + conf.getBoolean(ScmConfigKeys.HDDS_DATANODE_UPGRADE_LAYOUT_INLINE, + ScmConfigKeys.HDDS_DATANODE_UPGRADE_LAYOUT_INLINE_DEFAULT); + LOG.info("Running in upgrade mode:{}", this.isInUpgradeMode); } + + private File getClusterDir() { + File hddsVolumeRootDir = hddsVolume.getHddsRootDir(); + File clusterDir = new File(hddsVolumeRootDir, hddsVolume.getClusterID()); + return clusterDir; + } @Override public void run() { try { readVolume(hddsVolumeDir); - } catch (RuntimeException ex) { + } catch (Throwable t) { LOG.error("Caught a Run time exception during reading container files" + - " from Volume {} {}", hddsVolumeDir, ex); + " from Volume {} {}", hddsVolumeDir, t); Review comment: The exception msg needs to be fixed. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
