leesf commented on a change in pull request #1436: [HUDI-711] Refactor exporter
main logic
URL: https://github.com/apache/incubator-hudi/pull/1436#discussion_r397585126
##########
File path:
hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieSnapshotExporter.java
##########
@@ -159,18 +161,85 @@ public void testExportAsHudi() throws IOException {
assertTrue(dfs.exists(new Path(partition +
"/.hoodie_partition_metadata")));
assertTrue(dfs.exists(new Path(targetPath + "/_SUCCESS")));
}
+ }
+
+ public static class TestHoodieSnapshotExporterForEarlyAbort extends
ExporterTestHarness {
+
+ private HoodieSnapshotExporter.Config cfg;
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ cfg = new Config();
+ cfg.sourceBasePath = sourcePath;
+ cfg.targetOutputPath = targetPath;
+ cfg.outputFormat = OutputFormatValidator.HUDI;
+ }
@Test
- public void testExportEmptyDataset() throws IOException {
+ public void testExportWhenTargetPathExists() throws IOException {
+ // make target output path present
+ dfs.mkdirs(new Path(targetPath));
+
+ // export
+ Throwable t = null;
+ try {
+ new HoodieSnapshotExporter().export(jsc, cfg);
+ } catch (Exception e) {
+ t = e;
+ } finally {
+ assertNotNull(t);
+ assertTrue(t instanceof HoodieSnapshotExporterException);
+ assertEquals("The target output path already exists.", t.getMessage());
+ }
+ }
+
+ @Test
+ public void testExportDatasetWithNoCommit() throws IOException {
+ // delete commit files
+ List<Path> commitFiles = Arrays.stream(dfs.listStatus(new
Path(sourcePath + "/.hoodie")))
+ .map(FileStatus::getPath)
+ .filter(filePath -> filePath.getName().endsWith(".commit"))
+ .collect(Collectors.toList());
+ for (Path p : commitFiles) {
+ dfs.delete(p, false);
+ }
+
+ // export
+ Throwable t = null;
+ try {
+ new HoodieSnapshotExporter().export(jsc, cfg);
+ } catch (Exception e) {
+ t = e;
+ } finally {
+ assertNotNull(t);
+ assertTrue(t instanceof HoodieSnapshotExporterException);
+ assertEquals("No commits present. Nothing to snapshot.",
t.getMessage());
+ }
+
+ // Check results
+ assertFalse(dfs.exists(new Path(targetPath)));
+ }
+
+ @Test
+ public void testExportDatasetWithNoPartition() throws IOException {
// delete all source data
dfs.delete(new Path(sourcePath + "/" + PARTITION_PATH), true);
// export
- new
HoodieSnapshotExporter().export(SparkSession.builder().config(jsc.getConf()).getOrCreate(),
cfg);
+ Throwable t = null;
+ try {
+ new HoodieSnapshotExporter().export(jsc, cfg);
+ } catch (Exception e) {
+ t = e;
+ } finally {
+ assertNotNull(t);
+ assertTrue(t instanceof HoodieSnapshotExporterException);
+ assertEquals("The source dataset has 0 partition to snapshot.",
t.getMessage());
Review comment:
ditto
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
