GeorgeJahad commented on code in PR #3980:
URL: https://github.com/apache/ozone/pull/3980#discussion_r1131560223
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java:
##########
@@ -259,69 +291,259 @@ public void testSpnegoEnabled() throws Exception {
}
@Test
- public void testWriteCheckpointToOutputStream() throws Exception {
-
- FileInputStream fis = null;
- FileOutputStream fos = null;
-
- try {
- String testDirName = folder.newFolder().getAbsolutePath();
- File file = new File(testDirName + "/temp1.txt");
- OutputStreamWriter writer = new OutputStreamWriter(
- new FileOutputStream(file), StandardCharsets.UTF_8);
- writer.write("Test data 1");
- writer.close();
-
- file = new File(testDirName + "/temp2.txt");
- writer = new OutputStreamWriter(
- new FileOutputStream(file), StandardCharsets.UTF_8);
- writer.write("Test data 2");
- writer.close();
-
- File outputFile =
- new File(Paths.get(testDirName, "output_file.tgz").toString());
- TestDBCheckpoint dbCheckpoint = new TestDBCheckpoint(
- Paths.get(testDirName));
- writeDBCheckpointToStream(dbCheckpoint,
- new FileOutputStream(outputFile));
- assertNotNull(outputFile);
- } finally {
- IOUtils.closeStream(fis);
- IOUtils.closeStream(fos);
+ public void testWriteDbDataToStream()
+ throws Exception {
+ prepSnapshotData();
+ // set http param to include snapshot data
+ when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA))
+ .thenReturn("true");
+
+ // get the tarball
+ try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) {
+ omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, requestMock,
+ fileOutputStream);
+ }
+
+ // Untar the file into a temp folder to be examined
+ String testDirName = folder.newFolder().getAbsolutePath();
+ int testDirLength = testDirName.length() + 1;
+ String newDbDirName = testDirName + OM_KEY_PREFIX + OM_DB_NAME;
+ int newDbDirLength = newDbDirName.length() + 1;
+ File newDbDir = new File(newDbDirName);
+ newDbDir.mkdirs();
+ FileUtil.unTar(tempFile, newDbDir);
+
+ // Move snapshot dir to correct location
+ new File(newDbDirName, OM_SNAPSHOT_DIR)
+ .renameTo(new File(newDbDir.getParent(), OM_SNAPSHOT_DIR));
+
+
+ // Confirm the checkpoint directories match, (after remove extras)
+ Path checkpointLocation = dbCheckpoint.getCheckpointLocation();
+ Set<String> initialCheckpointSet = getFiles(checkpointLocation,
+ checkpointLocation.toString().length() + 1);
+ Path finalCheckpointLocation = Paths.get(newDbDirName);
+ Set<String> finalCheckpointSet = getFiles(finalCheckpointLocation,
+ newDbDirLength);
+
+ Assert.assertTrue("hardlink file exists in checkpoint dir",
+ finalCheckpointSet.contains(OM_HARDLINK_FILE));
+ finalCheckpointSet.remove(OM_HARDLINK_FILE);
+ Assert.assertEquals(initialCheckpointSet, finalCheckpointSet);
+
+ int metaDirLength = metaDir.toString().length() + 1;
+ String shortSnapshotLocation =
+ truncateFileName(metaDirLength, Paths.get(snapshotDirName));
+ String shortSnapshotLocation2 =
+ truncateFileName(metaDirLength, Paths.get(snapshotDirName2));
+ String shortCompactionDirLocation =
+ truncateFileName(metaDirLength, compactionDirPath);
+
+ Set<String> finalFullSet =
+ getFiles(Paths.get(testDirName, OM_SNAPSHOT_DIR), testDirLength);
+
+ // check each line in the hard link file
+ Stream<String> lines = Files.lines(Paths.get(newDbDirName,
+ OM_HARDLINK_FILE));
+
+ List<String> fabricatedLinkLines = new ArrayList<>();
+ for (String line: lines.collect(Collectors.toList())) {
+ Assert.assertFalse("CURRENT file is not a hard link",
+ line.contains("CURRENT"));
+ if (line.contains("fabricatedFile")) {
+ fabricatedLinkLines.add(line);
+ } else {
+ checkLine(shortSnapshotLocation, shortSnapshotLocation2, line);
+ // add links to the final set
+ finalFullSet.add(line.split("\t")[0]);
+ }
}
+
+ Set<String> directories = Sets.newHashSet(
+ shortSnapshotLocation, shortSnapshotLocation2,
+ shortCompactionDirLocation);
+ checkFabricatedLines(directories, fabricatedLinkLines, testDirName);
+
+ Set<String> initialFullSet =
+ getFiles(Paths.get(metaDir.toString(), OM_SNAPSHOT_DIR),
metaDirLength);
+ Assert.assertEquals("found expected snapshot files",
+ initialFullSet, finalFullSet);
}
-}
-class TestDBCheckpoint implements DBCheckpoint {
+ @Test
+ public void testWriteDbDataWithoutOmSnapshot()
+ throws Exception {
+ prepSnapshotData();
+
+ // set http param to exclude snapshot data
+ when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA))
+ .thenReturn(null);
+
+ // get the tarball
+ try (FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) {
+ omDbCheckpointServletMock.writeDbDataToStream(dbCheckpoint, requestMock,
+ fileOutputStream);
+ }
+
+ // Untar the file into a temp folder to be examined
+ String testDirName = folder.newFolder().getAbsolutePath();
+ int testDirLength = testDirName.length() + 1;
+ FileUtil.unTar(tempFile, new File(testDirName));
- private final Path checkpointFile;
+ // Confirm the checkpoint directories match
+ Path checkpointLocation = dbCheckpoint.getCheckpointLocation();
+ Set<String> initialCheckpointSet = getFiles(checkpointLocation,
+ checkpointLocation.toString().length() + 1);
+ Path finalCheckpointLocation = Paths.get(testDirName);
+ Set<String> finalCheckpointSet = getFiles(finalCheckpointLocation,
+ testDirLength);
- TestDBCheckpoint(Path checkpointFile) {
- this.checkpointFile = checkpointFile;
+ Assert.assertEquals(initialCheckpointSet, finalCheckpointSet);
}
- @Override
- public Path getCheckpointLocation() {
- return checkpointFile;
+
+
+ private void prepSnapshotData() throws Exception {
+ setupCluster();
+ metaDir = OMStorage.getOmDbDir(conf);
+
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+
+ // Create dummy keys for snapshotting
+ TestDataUtil.createKey(bucket, UUID.randomUUID().toString(),
+ "content");
+ TestDataUtil.createKey(bucket, UUID.randomUUID().toString(),
+ "content");
+
+ // this sleep can be removed after this is fixed:
+ // https://issues.apache.org/jira/browse/HDDS-7279
+ Thread.sleep(2000);
+ snapshotDirName =
+ createSnapshot(bucket.getVolumeName(), bucket.getName());
+ snapshotDirName2 =
+ createSnapshot(bucket.getVolumeName(), bucket.getName());
+
+
+ // create fabricated links to snapshot dirs
+ // to confirm that links are recognized even if
+ // they are don't point to the checkpoint directory
+ Path fabricatedFile = Paths.get(snapshotDirName, "fabricatedFile");
+ Path fabricatedLink = Paths.get(snapshotDirName2, "fabricatedFile");
+
+ Files.write(fabricatedFile,
+ "fabricatedData".getBytes(StandardCharsets.UTF_8));
+ Files.createLink(fabricatedLink, fabricatedFile);
+
+ // simulate links from the compaction dir
+ compactionDirPath = Paths.get(metaDir.toString(),
+ OM_SNAPSHOT_DIFF_DIR, OM_COMPACTION_BACKUP_DIR);
+ Path fabricatedLink2 = Paths.get(compactionDirPath.toString(),
+ "fabricatedFile");
+ Files.createLink(fabricatedLink2, fabricatedFile);
+ Path currentFile = Paths.get(metaDir.toString(),
+ OM_DB_NAME, "CURRENT");
+ Path currentLink = Paths.get(compactionDirPath.toString(), "CURRENT");
+ Files.createLink(currentLink, currentFile);
+
+ dbCheckpoint = cluster.getOzoneManager()
+ .getMetadataManager().getStore()
+ .getCheckpoint(true);
+
}
- @Override
- public long getCheckpointTimestamp() {
- return 0;
+ private String createSnapshot(String vname, String bname)
+ throws IOException, InterruptedException, TimeoutException {
+ final OzoneManager om = cluster.getOzoneManager();
+ String snapshotName = UUID.randomUUID().toString();
+ OzoneManagerProtocol writeClient = cluster.getRpcClient().getObjectStore()
+ .getClientProxy().getOzoneManagerClient();
+
+ writeClient.createSnapshot(vname, bname, snapshotName);
+ SnapshotInfo snapshotInfo = om.getMetadataManager().getSnapshotInfoTable()
+ .get(SnapshotInfo.getTableKey(vname, bname, snapshotName));
+ String snapshotPath = getSnapshotPath(conf, snapshotInfo)
+ + OM_KEY_PREFIX;
+ GenericTestUtils.waitFor(() -> new File(snapshotPath).exists(),
+ 100, 2000);
+ return snapshotPath;
}
- @Override
- public long getLatestSequenceNumber() {
- return 0;
+ private Set<String> getFiles(Path path, int truncateLength)
+ throws IOException {
+ return getFiles(path, truncateLength, new HashSet<>());
+ }
+
+ // Get all files below path, recursively, (skipping fabricated files)
+ @SuppressFBWarnings({"NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"})
+ private Set<String> getFiles(Path path, int truncateLength,
+ Set<String> fileSet) throws IOException {
+ try (Stream<Path> files = Files.list(path)) {
+ for (Path file : files.collect(Collectors.toList())) {
+ if (file.toFile().isDirectory()) {
+ getFiles(file, truncateLength, fileSet);
+ }
+ if (!file.getFileName().toString().equals("fabricatedFile")) {
+ fileSet.add(truncateFileName(truncateLength, file));
+ }
+ }
+ }
+ return fileSet;
}
- @Override
- public long checkpointCreationTimeTaken() {
- return 0;
+ // tests to see that fabricated link lines in hardlink file are
Review Comment:
changed to java doc
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
