errose28 commented on code in PR #8154: URL: https://github.com/apache/ozone/pull/8154#discussion_r2025973141
########## hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRatisLogRepair.java: ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.repair.om; + +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.EchoRPC; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.repair.RepairTool; +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.server.metrics.SegmentedRaftLogMetrics; +import org.apache.ratis.server.raftlog.segmented.LogSegmentPath; +import org.apache.ratis.server.raftlog.segmented.LogSegmentStartEnd; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogInputStream; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream; +import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.SizeInBytes; +import picocli.CommandLine; + + +/** + * Tool to omit a raft log in a ratis segment file. + */ [email protected]( + name = "ratislogrepair", Review Comment: IMO `skip-ratis-transaction` or something similar is more descriptive, since this isn't doing a general purpose repair. We may add other repair tools for different aspects of the Ratis log in the future if needed. ########## hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRatisLogRepair.java: ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.repair.om; + +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.EchoRPC; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.repair.RepairTool; +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.server.metrics.SegmentedRaftLogMetrics; +import org.apache.ratis.server.raftlog.segmented.LogSegmentPath; +import org.apache.ratis.server.raftlog.segmented.LogSegmentStartEnd; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogInputStream; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream; +import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.SizeInBytes; +import picocli.CommandLine; + + +/** + * Tool to omit a raft log in a ratis segment file. + */ [email protected]( + name = "ratislogrepair", + description = "CLI to omit a raft log in a ratis segment file. " + + "Requires admin privileges.", Review Comment: The description should also provide usage recommendations. For example, the command needs to be done for the same transaction on all 3 OMs only when they are all crashing applying the same transaction. If there is only one OM that is crashing, it should have the DB manually copied from one of the good OMs instead. ########## hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRatisLogRepair.java: ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.repair.om; + +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.EchoRPC; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.repair.RepairTool; +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.server.metrics.SegmentedRaftLogMetrics; +import org.apache.ratis.server.raftlog.segmented.LogSegmentPath; +import org.apache.ratis.server.raftlog.segmented.LogSegmentStartEnd; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogInputStream; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream; +import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.SizeInBytes; +import picocli.CommandLine; + + +/** + * Tool to omit a raft log in a ratis segment file. + */ [email protected]( + name = "ratislogrepair", + description = "CLI to omit a raft log in a ratis segment file. " + + "Requires admin privileges.", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class +) +public class OMRatisLogRepair extends RepairTool { + + @CommandLine.Option(names = {"-s", "--segment-path", "--segmentPath"}, + required = true, + description = "Path of the input segment file") + private File segmentFile; + @CommandLine.Option(names = {"-o", "--output-path", "--outputPath"}, + required = true, + description = "Path of the repaired segment file") + private String outputPath; + + @CommandLine.Option(names = {"--index"}, + required = true, + description = "Path of the segment file") + private long index; + + @Override + public void execute() throws Exception { + + LogSegmentPath pi = LogSegmentPath.matchLogSegment(this.segmentFile.toPath()); + if (pi == null) { + error("Invalid segment file"); + throw new RuntimeException("Invalid Segment File"); + } + info("Processing Raft Log file: " + this.segmentFile.getAbsolutePath() + " size:" + this.segmentFile.length()); + SegmentedRaftLogOutputStream outputStream = null; + SegmentedRaftLogInputStream logInputStream = null; + + try { + logInputStream = getInputStream(pi); + File outputFile = createOutputFile(outputPath); + outputStream = new SegmentedRaftLogOutputStream(outputFile, false, + 1024, 1024, ByteBuffer.allocateDirect(10 * 1024)); + + RaftProtos.LogEntryProto next; + for (RaftProtos.LogEntryProto prev = null; (next = logInputStream.nextEntry()) != null; prev = next) { + if (prev != null) { + Preconditions.assertTrue(next.getIndex() == prev.getIndex() + 1L, + "gap between entry %s and entry %s", prev, next); + } + + if (next.getIndex() != index) { + // all other logs will be written as it is + outputStream.write(next); Review Comment: We should add support for `--dry-run` too. ########## hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRatisLogRepair.java: ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.repair.om; + +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.EchoRPC; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.repair.RepairTool; +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.server.metrics.SegmentedRaftLogMetrics; +import org.apache.ratis.server.raftlog.segmented.LogSegmentPath; +import org.apache.ratis.server.raftlog.segmented.LogSegmentStartEnd; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogInputStream; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream; +import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.SizeInBytes; +import picocli.CommandLine; + + +/** + * Tool to omit a raft log in a ratis segment file. + */ [email protected]( + name = "ratislogrepair", + description = "CLI to omit a raft log in a ratis segment file. " + + "Requires admin privileges.", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class +) +public class OMRatisLogRepair extends RepairTool { + + @CommandLine.Option(names = {"-s", "--segment-path", "--segmentPath"}, + required = true, + description = "Path of the input segment file") + private File segmentFile; + @CommandLine.Option(names = {"-o", "--output-path", "--outputPath"}, + required = true, + description = "Path of the repaired segment file") + private String outputPath; + + @CommandLine.Option(names = {"--index"}, + required = true, + description = "Path of the segment file") + private long index; Review Comment: How would you know which of the multiple transactions to update in practice? Usually what happens is the OM crashes on one transaction, and you skip that one only. Maybe you could proactively go through with the ratis log parser and try to find other instances, but I think the usual case would be to just turn on the OM and see if it works. ########## hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRatisLogRepair.java: ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.repair.om; + +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.EchoRPC; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.repair.RepairTool; +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.server.metrics.SegmentedRaftLogMetrics; +import org.apache.ratis.server.raftlog.segmented.LogSegmentPath; +import org.apache.ratis.server.raftlog.segmented.LogSegmentStartEnd; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogInputStream; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream; +import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.SizeInBytes; +import picocli.CommandLine; + + +/** + * Tool to omit a raft log in a ratis segment file. + */ [email protected]( + name = "ratislogrepair", + description = "CLI to omit a raft log in a ratis segment file. " + + "Requires admin privileges.", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class +) +public class OMRatisLogRepair extends RepairTool { + + @CommandLine.Option(names = {"-s", "--segment-path", "--segmentPath"}, + required = true, + description = "Path of the input segment file") + private File segmentFile; + @CommandLine.Option(names = {"-o", "--output-path", "--outputPath"}, + required = true, + description = "Path of the repaired segment file") + private String outputPath; Review Comment: Can we simplify the usage by: - Taking as input the path to the Ratis log directory, and having the tool find the correct segment file. - Making the argument take a path to put the backup of the original segment file and having the segment file be written into the final location by default. - This way no further steps are required in the common case, but the original is still preserved. ########## hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRatisLogRepair.java: ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.repair.om; + +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.EchoRPC; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.repair.RepairTool; +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.server.metrics.SegmentedRaftLogMetrics; +import org.apache.ratis.server.raftlog.segmented.LogSegmentPath; +import org.apache.ratis.server.raftlog.segmented.LogSegmentStartEnd; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogInputStream; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream; +import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.SizeInBytes; +import picocli.CommandLine; + + +/** + * Tool to omit a raft log in a ratis segment file. + */ [email protected]( + name = "ratislogrepair", + description = "CLI to omit a raft log in a ratis segment file. " + + "Requires admin privileges.", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class +) +public class OMRatisLogRepair extends RepairTool { + + @CommandLine.Option(names = {"-s", "--segment-path", "--segmentPath"}, + required = true, + description = "Path of the input segment file") + private File segmentFile; + @CommandLine.Option(names = {"-o", "--output-path", "--outputPath"}, + required = true, + description = "Path of the repaired segment file") + private String outputPath; + + @CommandLine.Option(names = {"--index"}, + required = true, + description = "Path of the segment file") + private long index; + + @Override + public void execute() throws Exception { + + LogSegmentPath pi = LogSegmentPath.matchLogSegment(this.segmentFile.toPath()); + if (pi == null) { + error("Invalid segment file"); + throw new RuntimeException("Invalid Segment File"); Review Comment: I think this prints the message to stderr twice, right? ########## hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRatisLogRepair.java: ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.repair.om; + +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.EchoRPC; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.repair.RepairTool; +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.server.metrics.SegmentedRaftLogMetrics; +import org.apache.ratis.server.raftlog.segmented.LogSegmentPath; +import org.apache.ratis.server.raftlog.segmented.LogSegmentStartEnd; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogInputStream; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream; +import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.SizeInBytes; +import picocli.CommandLine; + + +/** + * Tool to omit a raft log in a ratis segment file. + */ [email protected]( + name = "ratislogrepair", + description = "CLI to omit a raft log in a ratis segment file. " + + "Requires admin privileges.", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class +) +public class OMRatisLogRepair extends RepairTool { + + @CommandLine.Option(names = {"-s", "--segment-path", "--segmentPath"}, + required = true, + description = "Path of the input segment file") + private File segmentFile; + @CommandLine.Option(names = {"-o", "--output-path", "--outputPath"}, + required = true, + description = "Path of the repaired segment file") + private String outputPath; + + @CommandLine.Option(names = {"--index"}, + required = true, + description = "Path of the segment file") Review Comment: ```suggestion description = "Index of the failing transaction that should be removed") ``` ########## hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRatisLogRepair.java: ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.repair.om; + +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.EchoRPC; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.repair.RepairTool; +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.server.metrics.SegmentedRaftLogMetrics; +import org.apache.ratis.server.raftlog.segmented.LogSegmentPath; +import org.apache.ratis.server.raftlog.segmented.LogSegmentStartEnd; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogInputStream; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream; +import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.SizeInBytes; +import picocli.CommandLine; + + +/** + * Tool to omit a raft log in a ratis segment file. + */ [email protected]( + name = "ratislogrepair", + description = "CLI to omit a raft log in a ratis segment file. " + + "Requires admin privileges.", Review Comment: This should be an offline tool, so the permissions would be to ssh into the server and read the file, not Ozone admin permissions. We should also specify in the description that this is an offline command. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
