swamirishi commented on code in PR #9150:
URL: https://github.com/apache/ozone/pull/9150#discussion_r2483103605
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java:
##########
@@ -253,8 +265,32 @@ void addVersionNodeWithDependents(OmSnapshotLocalData
snapshotLocalData) throws
}
}
- private void init() throws IOException {
+ private void incrementOrphanCheckCount(UUID snapshotId) {
+ if (snapshotId != null) {
+ this.snapshotToBeCheckedForOrphans.compute(snapshotId, (k, v) -> v ==
null ? 1 : (v + 1));
+ }
+ }
+
+ private void decrementOrphanCheckCount(UUID snapshotId, int decrementBy) {
+ this.snapshotToBeCheckedForOrphans.compute(snapshotId, (k, v) -> {
+ if (v == null) {
+ return null;
+ }
+ int newValue = v - decrementBy;
+ if (newValue <= 0) {
+ return null;
+ }
+ return newValue;
+ });
+ }
+
+ Map<UUID, Integer> getSnapshotToBeCheckedForOrphans() {
+ return snapshotToBeCheckedForOrphans;
Review Comment:
added Visible for testing annotation
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java:
##########
@@ -694,47 +795,76 @@ public synchronized void commit() throws IOException {
// Need to update the disk state if and only if the dirty bit is set.
if (isDirty()) {
String filePath = getSnapshotLocalPropertyYamlPath(super.snapshotId);
- String tmpFilePath = filePath + ".tmp";
- File tmpFile = new File(tmpFilePath);
- boolean tmpFileExists = tmpFile.exists();
- if (tmpFileExists) {
- tmpFileExists = !tmpFile.delete();
- }
- if (tmpFileExists) {
- throw new IOException("Unable to delete tmp file " + tmpFilePath);
+ File snapshotLocalDataFile = new File(filePath);
+ if (!localDataVersionNodes.getSnapshotVersions().isEmpty()) {
+ String tmpFilePath = filePath + ".tmp";
+ File tmpFile = new File(tmpFilePath);
+ boolean tmpFileExists = tmpFile.exists();
+ if (tmpFileExists) {
+ tmpFileExists = !tmpFile.delete();
+ }
+ if (tmpFileExists) {
+ throw new IOException("Unable to delete tmp file " + tmpFilePath);
+ }
+ snapshotLocalDataSerializer.save(new File(tmpFilePath),
super.snapshotLocalData);
+ Files.move(tmpFile.toPath(), Paths.get(filePath),
StandardCopyOption.ATOMIC_MOVE,
+ StandardCopyOption.REPLACE_EXISTING);
+ } else if (snapshotLocalDataFile.exists()) {
+ LOG.info("Deleting Yaml file corresponding to snapshotId: {} in path
: {}",
+ super.snapshotId, snapshotLocalDataFile.getAbsolutePath());
+ if (!snapshotLocalDataFile.delete()) {
+ throw new IOException("Unable to delete file " +
snapshotLocalDataFile.getAbsolutePath());
+ }
}
- snapshotLocalDataSerializer.save(new File(tmpFilePath),
super.snapshotLocalData);
- Files.move(tmpFile.toPath(), Paths.get(filePath),
StandardCopyOption.ATOMIC_MOVE,
- StandardCopyOption.REPLACE_EXISTING);
- upsertNode(super.snapshotId, localDataVersionNodes);
+ upsertNode(super.snapshotId, localDataVersionNodes,
getSnapshotLocalData().getTransactionInfo() != null);
// Reset dirty bit
resetDirty();
}
}
- private void upsertNode(UUID snapshotId, SnapshotVersionsMeta
snapshotVersions) throws IOException {
+ private void upsertNode(UUID snapshotId, SnapshotVersionsMeta
snapshotVersions,
+ boolean transactionInfoSet) throws IOException {
internalLock.writeLock().lock();
try {
SnapshotVersionsMeta existingSnapVersions =
getVersionNodeMap().remove(snapshotId);
Map<Integer, LocalDataVersionNode> existingVersions =
existingSnapVersions == null ? Collections.emptyMap() :
existingSnapVersions.getSnapshotVersions();
+ Map<Integer, LocalDataVersionNode> newVersions =
snapshotVersions.getSnapshotVersions();
Map<Integer, List<LocalDataVersionNode>> predecessors = new
HashMap<>();
+ boolean versionsRemoved = false;
// Track all predecessors of the existing versions and remove the node
from the graph.
for (Map.Entry<Integer, LocalDataVersionNode> existingVersion :
existingVersions.entrySet()) {
LocalDataVersionNode existingVersionNode =
existingVersion.getValue();
// Create a copy of predecessors since the list of nodes returned
would be a mutable set and it changes as the
// nodes in the graph would change.
predecessors.put(existingVersion.getKey(), new
ArrayList<>(localDataGraph.predecessors(existingVersionNode)));
+ versionsRemoved = versionsRemoved ||
!newVersions.containsKey(existingVersion.getKey());
localDataGraph.removeNode(existingVersionNode);
}
+
// Add the nodes to be added in the graph and map.
addSnapshotVersionMeta(snapshotId, snapshotVersions);
// Reconnect all the predecessors for existing nodes.
- for (Map.Entry<Integer, LocalDataVersionNode> entry :
snapshotVersions.getSnapshotVersions().entrySet()) {
+ for (Map.Entry<Integer, LocalDataVersionNode> entry :
newVersions.entrySet()) {
for (LocalDataVersionNode predecessor :
predecessors.getOrDefault(entry.getKey(), Collections.emptyList())) {
localDataGraph.putEdge(predecessor, entry.getValue());
}
}
+ if (existingSnapVersions != null) {
+ // The previous snapshotId could have become an orphan entry or
could have orphan versions.(In case of
+ // version removals)
+ if (versionsRemoved ||
!Objects.equals(existingSnapVersions.getPreviousSnapshotId(),
+ snapshotVersions.getPreviousSnapshotId())) {
+
incrementOrphanCheckCount(existingSnapVersions.getPreviousSnapshotId());
+ }
+ // If the transactionInfo set this means the snapshot has been
purged and the entire yaml file could have
+ // become an orphan if the version is also updated it
+ // could mean that there could be some orphan version present within
the
+ // same snapshot.
Review Comment:
Added comment
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java:
##########
@@ -694,47 +795,76 @@ public synchronized void commit() throws IOException {
// Need to update the disk state if and only if the dirty bit is set.
if (isDirty()) {
String filePath = getSnapshotLocalPropertyYamlPath(super.snapshotId);
- String tmpFilePath = filePath + ".tmp";
- File tmpFile = new File(tmpFilePath);
- boolean tmpFileExists = tmpFile.exists();
- if (tmpFileExists) {
- tmpFileExists = !tmpFile.delete();
- }
- if (tmpFileExists) {
- throw new IOException("Unable to delete tmp file " + tmpFilePath);
+ File snapshotLocalDataFile = new File(filePath);
+ if (!localDataVersionNodes.getSnapshotVersions().isEmpty()) {
+ String tmpFilePath = filePath + ".tmp";
+ File tmpFile = new File(tmpFilePath);
+ boolean tmpFileExists = tmpFile.exists();
+ if (tmpFileExists) {
+ tmpFileExists = !tmpFile.delete();
+ }
+ if (tmpFileExists) {
+ throw new IOException("Unable to delete tmp file " + tmpFilePath);
+ }
+ snapshotLocalDataSerializer.save(new File(tmpFilePath),
super.snapshotLocalData);
+ Files.move(tmpFile.toPath(), Paths.get(filePath),
StandardCopyOption.ATOMIC_MOVE,
+ StandardCopyOption.REPLACE_EXISTING);
+ } else if (snapshotLocalDataFile.exists()) {
+ LOG.info("Deleting Yaml file corresponding to snapshotId: {} in path
: {}",
+ super.snapshotId, snapshotLocalDataFile.getAbsolutePath());
+ if (!snapshotLocalDataFile.delete()) {
+ throw new IOException("Unable to delete file " +
snapshotLocalDataFile.getAbsolutePath());
+ }
}
- snapshotLocalDataSerializer.save(new File(tmpFilePath),
super.snapshotLocalData);
- Files.move(tmpFile.toPath(), Paths.get(filePath),
StandardCopyOption.ATOMIC_MOVE,
- StandardCopyOption.REPLACE_EXISTING);
- upsertNode(super.snapshotId, localDataVersionNodes);
+ upsertNode(super.snapshotId, localDataVersionNodes,
getSnapshotLocalData().getTransactionInfo() != null);
// Reset dirty bit
resetDirty();
}
}
- private void upsertNode(UUID snapshotId, SnapshotVersionsMeta
snapshotVersions) throws IOException {
+ private void upsertNode(UUID snapshotId, SnapshotVersionsMeta
snapshotVersions,
+ boolean transactionInfoSet) throws IOException {
internalLock.writeLock().lock();
try {
SnapshotVersionsMeta existingSnapVersions =
getVersionNodeMap().remove(snapshotId);
Map<Integer, LocalDataVersionNode> existingVersions =
existingSnapVersions == null ? Collections.emptyMap() :
existingSnapVersions.getSnapshotVersions();
+ Map<Integer, LocalDataVersionNode> newVersions =
snapshotVersions.getSnapshotVersions();
Map<Integer, List<LocalDataVersionNode>> predecessors = new
HashMap<>();
+ boolean versionsRemoved = false;
// Track all predecessors of the existing versions and remove the node
from the graph.
for (Map.Entry<Integer, LocalDataVersionNode> existingVersion :
existingVersions.entrySet()) {
LocalDataVersionNode existingVersionNode =
existingVersion.getValue();
// Create a copy of predecessors since the list of nodes returned
would be a mutable set and it changes as the
// nodes in the graph would change.
predecessors.put(existingVersion.getKey(), new
ArrayList<>(localDataGraph.predecessors(existingVersionNode)));
+ versionsRemoved = versionsRemoved ||
!newVersions.containsKey(existingVersion.getKey());
localDataGraph.removeNode(existingVersionNode);
}
+
// Add the nodes to be added in the graph and map.
addSnapshotVersionMeta(snapshotId, snapshotVersions);
// Reconnect all the predecessors for existing nodes.
- for (Map.Entry<Integer, LocalDataVersionNode> entry :
snapshotVersions.getSnapshotVersions().entrySet()) {
+ for (Map.Entry<Integer, LocalDataVersionNode> entry :
newVersions.entrySet()) {
for (LocalDataVersionNode predecessor :
predecessors.getOrDefault(entry.getKey(), Collections.emptyList())) {
localDataGraph.putEdge(predecessor, entry.getValue());
}
}
+ if (existingSnapVersions != null) {
+ // The previous snapshotId could have become an orphan entry or
could have orphan versions.(In case of
+ // version removals)
+ if (versionsRemoved ||
!Objects.equals(existingSnapVersions.getPreviousSnapshotId(),
+ snapshotVersions.getPreviousSnapshotId())) {
+
incrementOrphanCheckCount(existingSnapVersions.getPreviousSnapshotId());
+ }
+ // If the transactionInfo set this means the snapshot has been
purged and the entire yaml file could have
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
