lw309637554 commented on a change in pull request #2254:
URL: https://github.com/apache/hudi/pull/2254#discussion_r544442648
##########
File path:
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java
##########
@@ -999,6 +999,110 @@ private void verifyInsertOverwritePartitionHandling(int
batch1RecordsCount, int
verifyRecordsWritten(commitTime2, inserts2, statuses);
}
+ /**
+ * Test scenario of writing fewer file groups for first partition than
second an third partition.
+ */
+ @Test
+ public void verifyDeletePartitionsHandlingWithFewerRecordsFirstPartition()
throws Exception {
+ verifyDeletePartitionsHandling(1000, 3000, 3000);
+ }
+
+ /**
+ * Test scenario of writing similar number file groups in partition.
+ */
+ @Test
+ public void verifyDeletePartitionsHandlingWithSimilarNumberOfRecords()
throws Exception {
+ verifyDeletePartitionsHandling(3000, 3000, 3000);
+ }
+
+ /**
+ * Test scenario of writing more file groups for first partition than second
an third partition.
+ */
+ @Test
+ public void
verifyDeletePartitionsHandlingHandlingWithFewerRecordsSecondThirdPartition()
throws Exception {
+ verifyDeletePartitionsHandling(3000, 1000, 1000);
+ }
+
+ /**
+ * 1) Do write1 (upsert) with 'batch1RecordsCount' number of records for
first partition.
+ * 2) Do write2 (upsert) with 'batch2RecordsCount' number of records for
second partition.
+ * 3) Do write3 (upsert) with 'batch3RecordsCount' number of records for
third partition.
+ * 4) delete first partition and check result.
+ * 5) delete second and third partition and check result.
+ *
+ */
+ private void verifyDeletePartitionsHandling(int batch1RecordsCount, int
batch2RecordsCount, int batch3RecordsCount) throws Exception {
+ HoodieWriteConfig config = getSmallInsertWriteConfig(2000, false);
+ SparkRDDWriteClient client = getHoodieWriteClient(config, false);
+ dataGen = new HoodieTestDataGenerator();
+
+ // Do Inserts for DEFAULT_FIRST_PARTITION_PATH
+ String commitTime1 = "001";
+ client.startCommitWithTime(commitTime1);
+ List<HoodieRecord> inserts1 =
dataGen.generateInsertsForPartition(commitTime1, batch1RecordsCount,
DEFAULT_FIRST_PARTITION_PATH);
+ JavaRDD<HoodieRecord> insertRecordsRDD1 = jsc.parallelize(inserts1, 2);
+ List<WriteStatus> statuses = client.upsert(insertRecordsRDD1,
commitTime1).collect();
+ assertNoWriteErrors(statuses);
+ Set<String> batch1Buckets = statuses.stream().map(s ->
s.getFileId()).collect(Collectors.toSet());
+ verifyRecordsWritten(commitTime1, inserts1, statuses);
+
+ // Do Inserts for DEFAULT_SECOND_PARTITION_PATH
+ String commitTime2 = "002";
Review comment:
thanks , have abstract two method
insertPartitionRecordsWithCommit、deletePartitionWithCommit. And reuse it.
Now deletePartitionAction implement insertOverwriteAction , I think copy on
write can cover it. Add MOR is not very necessary.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
